<commit_before>package sqlquery

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/dustinblackman/tidalwave/logger"
	"github.com/tidwall/gjson"
	dry "github.com/ungerik/go-dry"
	sqlparser "github.com/youtube/vitess/go/vt/sqlparser"
)

const (
	// TypeCount specifies result is a count result
	TypeCount = "count"
	// TypeDistinct specifies result is a distinct result
	TypeDistinct = "distinct"
	// TypeCountDistinct specifies result is a count distinct result
	TypeCountDistinct = "count-distinct"
	// TypeSearch specifies specifies result is a search result
	TypeSearch = "search"
)

// List of operators that use the Regex field in QueryParam
var regexOperators = []string{"like", "ilike", "regexp"}

// A list of strings replaced in a query string before being passed to the parser to avoid parsing errors.
var stringReplacements = [][]string{
	{"-", "__DASH__"},
}

// QueryParam holds a single piece of a queries WHERE and SELECT statements to be processed on log lines
type QueryParam struct {
	IsInt          bool
	KeyPath        string
	Regex          *regexp.Regexp
	Operator       string
	ValInt         int
	ValIntArray    []int
	ValString      string
	ValStringArray []string
}

// QueryParams holds all the information for a given query such SELECT, FROM, and WHERE statements to be easily processed later.
type QueryParams struct {
	From []string // TODO: Rename to Froms

	AggrPath string
	Dates    []DateParam
	Queries  []QueryParam // TODO Rename to Where
	Selects  []QueryParam
	Type     string
}

// Youtube's SQL parser doesn't like some characters in parts of the query.
// We replace them in New, and them restore them here after parsing the sql parsers response.
func repairString(key string) string {
	for _, entry := range stringReplacements {
		key = strings.Replace(key, entry[1], entry[0], -1)
	}
	return key
}

func assignTypeFieldsToParam(param QueryParam, value string) QueryParam {
	if i, err := strconv.Atoi(value); err == nil {
		param.IsInt = true
		param.ValInt = i
	} else {
		param.ValString = repairString(stripQuotes(value))

		// Handles building the Regex field on param when a string is selected
		if dry.StringListContains(regexOperators, param.Operator) {
			regexString := ""
			if param.ValString[:1] == "%" && param.ValString[len(param.ValString)-1:] != "%" {
				regexString = "^" + param.ValString[1:]
			} else if param.ValString[:1] != "%" && param.ValString[len(param.ValString)-1:] == "%" {
				regexString = param.ValString[:len(param.ValString)-1] + "$"
			} else if param.ValString[:1] == "%" && param.ValString[len(param.ValString)-1:] == "%" {
				regexString = param.ValString[1 : len(param.ValString)-1]
			} else {
				regexString = param.ValString
			}

			if param.Operator == "regexp" {
				param.Operator = "ilike" // TODO: Temp workaround
				regexString = "(?i)" + regexString
			}
			param.Regex = regexp.MustCompile(regexString)
		}
	}
	return param
}

func handleCompareExpr(expr *sqlparser.ComparisonExpr) QueryParam {
	param := QueryParam{
		KeyPath:  repairString(stripQuotes(sqlparser.String(expr.Left))),
		Operator: expr.Operator,
	}

	right := sqlparser.String(expr.Right)
	if expr.Operator == "in" {
		arrayValues := strings.Split(right[1:len(right)-1], ", ")
		for _, val := range arrayValues {
			if i, err := strconv.Atoi(val); err == nil {
				param.IsInt = true
				param.ValIntArray = append(param.ValIntArray, i)
			} else {
				param.ValStringArray = append(param.ValStringArray, repairString(stripQuotes(val)))
			}
		}

		// We can't have mixed types wheh comparing arrays. We default to strings if not all values were convertable to
		// numbers
		if len(param.ValIntArray) > 0 && len(param.ValStringArray) != 0 {
			param.IsInt = false
			for _, val := range param.ValIntArray {
				param.ValStringArray = append(param.ValStringArray, string(val))
			}
			param.ValIntArray = []int{}
		}
	} else {
		param = assignTypeFieldsToParam(param, right)
	}

	return param
}

func handleAndExpr(expr *sqlparser.AndExpr) []QueryParam {
	params := []QueryParam{}

	for _, side := range []interface{}{expr.Left, expr.Right} {
		params = append(params, handleExpr(side)...)
	}
	return params
}

func handleRandExpr(expr *sqlparser.RangeCond) []QueryParam {
	keypath := repairString(stripQuotes(sqlparser.String(expr.Left)))
	fromQuery := assignTypeFieldsToParam(QueryParam{
		KeyPath:  keypath,
		Operator: ">=",
	}, sqlparser.String(expr.From))
	toQuery := assignTypeFieldsToParam(QueryParam{
		KeyPath:  keypath,
		Operator: "<=",
	}, sqlparser.String(expr.To))

	return []QueryParam{fromQuery, toQuery}
}

func handleExpr(entry interface{}) []QueryParam {
	params := []QueryParam{}
	switch expr := entry.(type) {
	case *sqlparser.AndExpr:
		params = append(params, handleAndExpr(expr)...)
	case *sqlparser.ComparisonExpr:
		params = append(params, handleCompareExpr(expr))
	case *sqlparser.RangeCond:
		params = append(params, handleRandExpr(expr)...)
	}

	return params
}

// ProcessLine interates through all Queries created during the query parsing returning a bool stating whether all matched.
func (qp *QueryParams) ProcessLine(line *[]byte) bool {
	matchMap := []bool{}
	for _, q := range qp.Queries {
		value := gjson.GetBytes(*line, q.KeyPath)
		if value.Type == 0 { // gjson way of saying key not found
			break
		}

		if q.IsInt && value.Type == gjson.Number {
			if ProcessInt(&q, int(value.Num)) {
				matchMap = append(matchMap, true)
			} else {
				break
			}
		} else {
			if ProcessString(&q, value.Str) {
				matchMap = append(matchMap, true)
			} else {
				break
			}
		}
	}

	return len(matchMap) == len(qp.Queries)
}

// New parses a query string and returns a newly created QueryParams struc holding all parsed data.
func New(queryString string) *QueryParams {
	logger.Logger.Debug("Query: " + queryString)
	queryParams := QueryParams{Type: TypeSearch} // Default is search. TODO Move to if statement

	// Fixes "date" breaking the parser by wrapping it in quotes
	queryString = strings.Replace(queryString, " date", " 'date'", -1)
	// Adds support for ~~.
	queryString = strings.Replace(queryString, " ~~ ", " like ", -1)
	// TODO This is a temporary workaround to have "ilike" support.
	// The sqlparser doesn't accept ilike, so we use rlike instead until a better solution comes around.
	queryString = strings.Replace(queryString, " ilike ", " rlike ", -1)
	// Replace characters that the SQL parser won't accept that will be reverted back after parsing
	for _, entry := range stringReplacements {
		queryString = strings.Replace(queryString, entry[0], entry[1], -1)
	}

	tree, err := sqlparser.Parse(queryString)
	if err != nil {
		logger.Logger.Error(err.Error())
	}
	queryTree := tree.(*sqlparser.Select)

	// Selects
	// Makes sure the selected keys we want exists in the line.
	logger.Logger.Debugf("Query Tree: %s", spew.Sdump(queryTree))
	for _, entry := range queryTree.SelectExprs {
		// TODO: Support star expression
		switch entry := entry.(type) {
		case *sqlparser.NonStarExpr:
			switch exp := entry.Expr.(type) {
			// Simple selects
			case *sqlparser.ColName:
				queryParams.Selects = append(queryParams.Selects, QueryParam{
					KeyPath:  repairString(sqlparser.String(exp)),
					Operator: "exists",
				})
			// DISTINCT()
			case *sqlparser.ParenExpr:
				keyPath := repairString(sqlparser.String(exp.Expr.(*sqlparser.ColName)))
				queryParams.Selects = append(queryParams.Selects, QueryParam{
					KeyPath:  keyPath,
					Operator: "exists",
				})

				// Where distinct is set
				if len(queryTree.Distinct) >= 8 {
					queryParams.Type = TypeDistinct
					queryParams.AggrPath = keyPath
				}
			// All other function expressions. COUNT(), COUNT(DISTINCT())
			case *sqlparser.FuncExpr:
				switch aggrPath := exp.Exprs[0].(*sqlparser.NonStarExpr).Expr.(type) {
				case sqlparser.ValTuple:
					queryParams.AggrPath = repairString(sqlparser.String(aggrPath[0].(*sqlparser.ColName)))
				case *sqlparser.ColName:
					queryParams.AggrPath = repairString(sqlparser.String(aggrPath))
				case *sqlparser.ParenExpr:
					// Fixes COUNT(DISTINCT())
					switch aggrPath := aggrPath.Expr.(type) {
					case sqlparser.ValTuple:
						queryParams.AggrPath = repairString(sqlparser.String(aggrPath[0].(*sqlparser.ColName)))
					case *sqlparser.ColName:
						queryParams.AggrPath = repairString(sqlparser.String(aggrPath))
					}
				}

				queryParams.Selects = append(queryParams.Selects, QueryParam{
					KeyPath:  queryParams.AggrPath,
					Operator: "exists",
				})

				// TODO This will need to get a bit more advance when we start adding other functions.
				name := sqlparser.String(exp.Name)
				if name == "count" && exp.Distinct {
					queryParams.Type = TypeCountDistinct
				} else {
					queryParams.Type = TypeCount
				}
			}
		}
	}

	// Froms
	for _, entry := range queryTree.From {
		queryParams.From = append(queryParams.From, repairString(sqlparser.String(entry)))
	}

	if queryTree.Where != nil {
		for _, entry := range handleExpr(queryTree.Where.Expr) {
			if entry.KeyPath == "date" {
				queryParams.Dates = append(queryParams.Dates, createDateParam(entry.ValString, entry.Operator))
			} else {
				queryParams.Queries = append(queryParams.Queries, entry)
			}
		}
	}

	queryParams.Queries = append(queryParams.Selects, queryParams.Queries...)
	logger.Logger.Debugf("Query Params: %s", spew.Sdump(queryParams))
	return &queryParams
}
<commit_msg>fix dot syntax<commit_after>package sqlquery

import (
	"regexp"
	"strconv"
	"strings"

	"github.com/davecgh/go-spew/spew"
	"github.com/dustinblackman/tidalwave/logger"
	"github.com/tidwall/gjson"
	dry "github.com/ungerik/go-dry"
	sqlparser "github.com/youtube/vitess/go/vt/sqlparser"
)

const (
	// TypeCount specifies result is a count result
	TypeCount = "count"
	// TypeDistinct specifies result is a distinct result
	TypeDistinct = "distinct"
	// TypeCountDistinct specifies result is a count distinct result
	TypeCountDistinct = "count-distinct"
	// TypeSearch specifies specifies result is a search result
	TypeSearch = "search"
)

// List of operators that use the Regex field in QueryParam
var regexOperators = []string{"like", "ilike", "regexp"}

// A list of strings replaced in a query string before being passed to the parser to avoid parsing errors.
var stringReplacements = [][]string{
	{"-", "__DASH__"},
	{".", "__DOT__"},
}

// QueryParam holds a single piece of a queries WHERE and SELECT statements to be processed on log lines
type QueryParam struct {
	IsInt          bool
	KeyPath        string
	Regex          *regexp.Regexp
	Operator       string
	ValInt         int
	ValIntArray    []int
	ValString      string
	ValStringArray []string
}

// QueryParams holds all the information for a given query such SELECT, FROM, and WHERE statements to be easily processed later.
type QueryParams struct {
	From []string // TODO: Rename to Froms

	AggrPath string
	Dates    []DateParam
	Queries  []QueryParam // TODO Rename to Where
	Selects  []QueryParam
	Type     string
}

// Youtube's SQL parser doesn't like some characters in parts of the query.
// We replace them in New, and them restore them here after parsing the sql parsers response.
func repairString(key string) string {
	for _, entry := range stringReplacements {
		key = strings.Replace(key, entry[1], entry[0], -1)
	}
	return key
}

func assignTypeFieldsToParam(param QueryParam, value string) QueryParam {
	if i, err := strconv.Atoi(value); err == nil {
		param.IsInt = true
		param.ValInt = i
	} else {
		param.ValString = repairString(stripQuotes(value))

		// Handles building the Regex field on param when a string is selected
		if dry.StringListContains(regexOperators, param.Operator) {
			regexString := ""
			if param.ValString[:1] == "%" && param.ValString[len(param.ValString)-1:] != "%" {
				regexString = "^" + param.ValString[1:]
			} else if param.ValString[:1] != "%" && param.ValString[len(param.ValString)-1:] == "%" {
				regexString = param.ValString[:len(param.ValString)-1] + "$"
			} else if param.ValString[:1] == "%" && param.ValString[len(param.ValString)-1:] == "%" {
				regexString = param.ValString[1 : len(param.ValString)-1]
			} else {
				regexString = param.ValString
			}

			if param.Operator == "regexp" {
				param.Operator = "ilike" // TODO: Temp workaround
				regexString = "(?i)" + regexString
			}
			param.Regex = regexp.MustCompile(regexString)
		}
	}
	return param
}

func handleCompareExpr(expr *sqlparser.ComparisonExpr) QueryParam {
	param := QueryParam{
		KeyPath:  repairString(stripQuotes(sqlparser.String(expr.Left))),
		Operator: expr.Operator,
	}

	right := sqlparser.String(expr.Right)
	if expr.Operator == "in" {
		arrayValues := strings.Split(right[1:len(right)-1], ", ")
		for _, val := range arrayValues {
			if i, err := strconv.Atoi(val); err == nil {
				param.IsInt = true
				param.ValIntArray = append(param.ValIntArray, i)
			} else {
				param.ValStringArray = append(param.ValStringArray, repairString(stripQuotes(val)))
			}
		}

		// We can't have mixed types wheh comparing arrays. We default to strings if not all values were convertable to
		// numbers
		if len(param.ValIntArray) > 0 && len(param.ValStringArray) != 0 {
			param.IsInt = false
			for _, val := range param.ValIntArray {
				param.ValStringArray = append(param.ValStringArray, string(val))
			}
			param.ValIntArray = []int{}
		}
	} else {
		param = assignTypeFieldsToParam(param, right)
	}

	return param
}

func handleAndExpr(expr *sqlparser.AndExpr) []QueryParam {
	params := []QueryParam{}

	for _, side := range []interface{}{expr.Left, expr.Right} {
		params = append(params, handleExpr(side)...)
	}
	return params
}

func handleRandExpr(expr *sqlparser.RangeCond) []QueryParam {
	keypath := repairString(stripQuotes(sqlparser.String(expr.Left)))
	fromQuery := assignTypeFieldsToParam(QueryParam{
		KeyPath:  keypath,
		Operator: ">=",
	}, sqlparser.String(expr.From))
	toQuery := assignTypeFieldsToParam(QueryParam{
		KeyPath:  keypath,
		Operator: "<=",
	}, sqlparser.String(expr.To))

	return []QueryParam{fromQuery, toQuery}
}

func handleExpr(entry interface{}) []QueryParam {
	params := []QueryParam{}
	switch expr := entry.(type) {
	case *sqlparser.AndExpr:
		params = append(params, handleAndExpr(expr)...)
	case *sqlparser.ComparisonExpr:
		params = append(params, handleCompareExpr(expr))
	case *sqlparser.RangeCond:
		params = append(params, handleRandExpr(expr)...)
	}

	return params
}

// ProcessLine interates through all Queries created during the query parsing returning a bool stating whether all matched.
func (qp *QueryParams) ProcessLine(line *[]byte) bool {
	matchMap := []bool{}
	for _, q := range qp.Queries {
		value := gjson.GetBytes(*line, q.KeyPath)
		if value.Type == 0 { // gjson way of saying key not found
			break
		}

		if q.IsInt && value.Type == gjson.Number {
			if ProcessInt(&q, int(value.Num)) {
				matchMap = append(matchMap, true)
			} else {
				break
			}
		} else {
			if ProcessString(&q, value.Str) {
				matchMap = append(matchMap, true)
			} else {
				break
			}
		}
	}

	return len(matchMap) == len(qp.Queries)
}

// New parses a query string and returns a newly created QueryParams struc holding all parsed data.
func New(queryString string) *QueryParams {
	logger.Logger.Debug("Query: " + queryString)
	queryParams := QueryParams{Type: TypeSearch} // Default is search. TODO Move to if statement

	// Fixes "date" breaking the parser by wrapping it in quotes
	queryString = strings.Replace(queryString, " date", " 'date'", -1)
	// Adds support for ~~.
	queryString = strings.Replace(queryString, " ~~ ", " like ", -1)
	// TODO This is a temporary workaround to have "ilike" support.
	// The sqlparser doesn't accept ilike, so we use rlike instead until a better solution comes around.
	queryString = strings.Replace(queryString, " ilike ", " rlike ", -1)
	// Replace characters that the SQL parser won't accept that will be reverted back after parsing
	for _, entry := range stringReplacements {
		queryString = strings.Replace(queryString, entry[0], entry[1], -1)
	}

	tree, err := sqlparser.Parse(queryString)
	if err != nil {
		logger.Logger.Error(err.Error())
	}
	queryTree := tree.(*sqlparser.Select)

	// Selects
	// Makes sure the selected keys we want exists in the line.
	logger.Logger.Debugf("Query Tree: %s", spew.Sdump(queryTree))
	for _, entry := range queryTree.SelectExprs {
		// TODO: Support star expression
		switch entry := entry.(type) {
		case *sqlparser.NonStarExpr:
			switch exp := entry.Expr.(type) {
			// Simple selects
			case *sqlparser.ColName:
				queryParams.Selects = append(queryParams.Selects, QueryParam{
					KeyPath:  repairString(sqlparser.String(exp)),
					Operator: "exists",
				})
			// DISTINCT()
			case *sqlparser.ParenExpr:
				keyPath := repairString(sqlparser.String(exp.Expr.(*sqlparser.ColName)))
				queryParams.Selects = append(queryParams.Selects, QueryParam{
					KeyPath:  keyPath,
					Operator: "exists",
				})

				// Where distinct is set
				if len(queryTree.Distinct) >= 8 {
					queryParams.Type = TypeDistinct
					queryParams.AggrPath = keyPath
				}
			// All other function expressions. COUNT(), COUNT(DISTINCT())
			case *sqlparser.FuncExpr:
				switch aggrPath := exp.Exprs[0].(*sqlparser.NonStarExpr).Expr.(type) {
				case sqlparser.ValTuple:
					queryParams.AggrPath = repairString(sqlparser.String(aggrPath[0].(*sqlparser.ColName)))
				case *sqlparser.ColName:
					queryParams.AggrPath = repairString(sqlparser.String(aggrPath))
				case *sqlparser.ParenExpr:
					// Fixes COUNT(DISTINCT())
					switch aggrPath := aggrPath.Expr.(type) {
					case sqlparser.ValTuple:
						queryParams.AggrPath = repairString(sqlparser.String(aggrPath[0].(*sqlparser.ColName)))
					case *sqlparser.ColName:
						queryParams.AggrPath = repairString(sqlparser.String(aggrPath))
					}
				}

				queryParams.Selects = append(queryParams.Selects, QueryParam{
					KeyPath:  queryParams.AggrPath,
					Operator: "exists",
				})

				// TODO This will need to get a bit more advance when we start adding other functions.
				name := sqlparser.String(exp.Name)
				if name == "count" && exp.Distinct {
					queryParams.Type = TypeCountDistinct
				} else {
					queryParams.Type = TypeCount
				}
			}
		}
	}

	// Froms
	for _, entry := range queryTree.From {
		queryParams.From = append(queryParams.From, repairString(sqlparser.String(entry)))
	}

	if queryTree.Where != nil {
		for _, entry := range handleExpr(queryTree.Where.Expr) {
			if entry.KeyPath == "date" {
				queryParams.Dates = append(queryParams.Dates, createDateParam(entry.ValString, entry.Operator))
			} else {
				queryParams.Queries = append(queryParams.Queries, entry)
			}
		}
	}

	queryParams.Queries = append(queryParams.Selects, queryParams.Queries...)
	logger.Logger.Debugf("Query Params: %s", spew.Sdump(queryParams))
	return &queryParams
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/julienschmidt/httprouter"

	"gitlab.com/faststack/machinestack/model"
)

// MachineList lists all machines of a user
func (h *Handler) MachineList(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	claims := r.Context().Value("user").(jwt.Token).Claims.(jwt.MapClaims)

	var machines []model.Machine
	if err := h.DB.Model(&machines).Where("user_id = ?", claims["id"]).Select(); err != nil {
		WriteInternalError(w, "session info: db error", err)
		return
	}

	WriteMany(w, http.StatusOK, &machines)
}
<commit_msg>jsonapi expects list of pointers<commit_after>package api

import (
	"net/http"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/julienschmidt/httprouter"

	"gitlab.com/faststack/machinestack/model"
)

// MachineList lists all machines of a user
func (h *Handler) MachineList(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
	claims := r.Context().Value("user").(jwt.Token).Claims.(jwt.MapClaims)

	var machines []*model.Machine
	if err := h.DB.Model(&machines).Where("user_id = ?", claims["id"]).Select(); err != nil {
		WriteInternalError(w, "session info: db error", err)
		return
	}

	WriteMany(w, http.StatusOK, &machines)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package file

import (
	"os"
	"syscall"
)

type File struct {
	fd   syscall.Handle // file descriptor number
	name string         // file name at Open time
}

func newFile(fd syscall.Handle, name string) *File {
	if fd == ^syscall.Handle(0) {
		return nil
	}
	return &File{fd, name}
}

var (
	Stdin  = newFile(syscall.Stdin, "/dev/stdin")
	Stdout = newFile(syscall.Stdout, "/dev/stdout")
	Stderr = newFile(syscall.Stderr, "/dev/stderr")
)

func OpenFile(name string, mode int, perm uint32) (file *File, err error) {
	r, e := syscall.Open(name, mode, perm)
	if e != 0 {
		err = os.Errno(e)
	}
	return newFile(r, name), err
}

const (
	O_RDONLY = syscall.O_RDONLY
	O_RDWR   = syscall.O_RDWR
	O_CREATE = syscall.O_CREAT
	O_TRUNC  = syscall.O_TRUNC
)

func Open(name string) (file *File, err error) {
	return OpenFile(name, O_RDONLY, 0)
}

func Create(name string) (file *File, err error) {
	return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
}

func (file *File) Close() error {
	if file == nil {
		return os.EINVAL
	}
	e := syscall.Close(file.fd)
	file.fd = syscall.InvalidHandle // so it can't be closed again
	if e != 0 {
		return os.Errno(e)
	}
	return nil
}

func (file *File) Read(b []byte) (ret int, err error) {
	if file == nil {
		return -1, os.EINVAL
	}
	r, e := syscall.Read(file.fd, b)
	if e != 0 {
		err = os.Errno(e)
	}
	return int(r), err
}

func (file *File) Write(b []byte) (ret int, err error) {
	if file == nil {
		return -1, os.EINVAL
	}
	r, e := syscall.Write(file.fd, b)
	if e != 0 {
		err = os.Errno(e)
	}
	return int(r), err
}

func (file *File) String() string {
	return file.name
}
<commit_msg>doc/progs: fix windows version to satisfy new error<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package file

import (
	"os"
	"syscall"
)

type File struct {
	fd   syscall.Handle // file descriptor number
	name string         // file name at Open time
}

func newFile(fd syscall.Handle, name string) *File {
	if fd == ^syscall.Handle(0) {
		return nil
	}
	return &File{fd, name}
}

var (
	Stdin  = newFile(syscall.Stdin, "/dev/stdin")
	Stdout = newFile(syscall.Stdout, "/dev/stdout")
	Stderr = newFile(syscall.Stderr, "/dev/stderr")
)

func OpenFile(name string, mode int, perm uint32) (file *File, err error) {
	r, err := syscall.Open(name, mode, perm)
	return newFile(r, name), err
}

const (
	O_RDONLY = syscall.O_RDONLY
	O_RDWR   = syscall.O_RDWR
	O_CREATE = syscall.O_CREAT
	O_TRUNC  = syscall.O_TRUNC
)

func Open(name string) (file *File, err error) {
	return OpenFile(name, O_RDONLY, 0)
}

func Create(name string) (file *File, err error) {
	return OpenFile(name, O_RDWR|O_CREATE|O_TRUNC, 0666)
}

func (file *File) Close() error {
	if file == nil {
		return os.EINVAL
	}
	err := syscall.Close(file.fd)
	file.fd = syscall.InvalidHandle // so it can't be closed again
	return err
}

func (file *File) Read(b []byte) (ret int, err error) {
	if file == nil {
		return -1, os.EINVAL
	}
	r, err := syscall.Read(file.fd, b)
	return int(r), err
}

func (file *File) Write(b []byte) (ret int, err error) {
	if file == nil {
		return -1, os.EINVAL
	}
	r, err := syscall.Write(file.fd, b)
	return int(r), err
}

func (file *File) String() string {
	return file.name
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build windows

package libdokan

import (
	"sync"

	"github.com/keybase/kbfs/dokan"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

type fileOpener interface {
	open(ctx context.Context, oc *openContext, path []string) (f dokan.File, isDir bool, err error)
	dokan.File
}

// FolderList is a node that can list all of the logged-in user's
// favorite top-level folders, on either a public or private basis.
type FolderList struct {
	emptyFile
	fs *FS
	// only accept public folders
	public bool

	mu      sync.Mutex
	folders map[string]fileOpener
}

// GetFileInformation for dokan.
func (*FolderList) GetFileInformation(*dokan.FileInfo) (*dokan.Stat, error) {
	return defaultDirectoryInformation()
}

func (fl *FolderList) reportErr(ctx context.Context,
	mode libkbfs.ErrorModeType, tlfName libkbfs.CanonicalTlfName, err error, cancelFn func()) {
	if cancelFn != nil {
		defer cancelFn()
	}
	if err == nil {
		fl.fs.log.CDebugf(ctx, "Request complete")
		return
	}

	fl.fs.config.Reporter().ReportErr(ctx, tlfName, fl.public, mode, err)
	// We just log the error as debug, rather than error, because it
	// might just indicate an expected error such as an ENOENT.
	//
	// TODO: Classify errors and escalate the logging level of the
	// important ones.
	fl.fs.log.CDebugf(ctx, err.Error())

}

// open tries to open the correct thing. Following aliases and deferring to
// Dir.open as necessary.
func (fl *FolderList) open(ctx context.Context, oc *openContext, path []string) (f dokan.File, isDir bool, err error) {
	fl.fs.log.CDebugf(ctx, "FL Lookup %#v public=%v", path, fl.public)
	if len(path) == 0 {
		return oc.returnDirNoCleanup(fl)
	}

	defer func() {
		fl.reportErr(ctx, libkbfs.ReadMode, libkbfs.CanonicalTlfName(path[0]), err, nil)
	}()

	for oc.reduceRedirectionsLeft() {
		name := path[0]

		if name == "desktop.ini" {
			fl.fs.log.CDebugf(ctx, "FL Lookup ignoring desktop.ini")
			return nil, false, dokan.ErrObjectNameNotFound
		}

		fl.mu.Lock()
		child, ok := fl.folders[name]
		fl.mu.Unlock()

		if ok {
			fl.fs.log.CDebugf(ctx, "FL Lookup recursing to child %q", name)
			return child.open(ctx, oc, path[1:])
		}

		if len(path) == 1 && oc.isCreateDirectory() && isNewFolderName(name) {
			fl.fs.log.CDebugf(ctx, "FL Lookup creating EmptyFolder for Explorer")
			e := &EmptyFolder{}
			fl.lockedAddChild(name, e)
			return e, true, nil
		}

		h, err := libkbfs.ParseTlfHandle(
			ctx, fl.fs.config.KBPKI(), name, fl.public)
		fl.fs.log.CDebugf(ctx, "FL Lookup continuing -> %v,%v", h, err)
		switch err := err.(type) {
		case nil:
			// No error.
			break

		case libkbfs.TlfNameNotCanonical:
			// Only permit Aliases to targets that contain no errors.
			if !fl.isValidAliasTarget(ctx, err.NameToTry) {
				fl.fs.log.CDebugf(ctx, "FL Refusing alias to non-valid target %q", err.NameToTry)
				return nil, false, dokan.ErrObjectNameNotFound
			}

			if len(path) == 1 && oc.isOpenReparsePoint() {
				// Non-canonical name.
				n := &Alias{
					canon: err.NameToTry,
				}
				return n, true, nil
			}
			path[0] = err.NameToTry
			continue

		case libkbfs.NoSuchNameError, libkbfs.BadTLFNameError:
			return nil, false, dokan.ErrObjectNameNotFound

		default:
			// Some other error.
			return nil, false, err
		}

		fl.fs.log.CDebugf(ctx, "FL Lookup adding new child")
		child = newTLF(fl, h)
		fl.lockedAddChild(name, child)
		return child.open(ctx, oc, path[1:])
	}
	return nil, false, dokan.ErrObjectNameNotFound
}

func (fl *FolderList) forgetFolder(folderName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	delete(fl.folders, folderName)
}

// FindFiles for dokan.
func (fl *FolderList) FindFiles(fi *dokan.FileInfo, callback func(*dokan.NamedStat) error) (err error) {
	ctx, cancel := NewContextWithOpID(fl.fs, "FL FindFiles")
	defer func() { fl.fs.reportErr(ctx, libkbfs.ReadMode, err, cancel) }()

	_, _, err = fl.fs.config.KBPKI().GetCurrentUserInfo(ctx)
	isLoggedIn := err == nil

	var favs []libkbfs.Favorite
	if isLoggedIn {
		favs, err = fl.fs.config.KBFSOps().GetFavorites(ctx)
		if err != nil {
			return err
		}
	}
	var ns dokan.NamedStat
	ns.FileAttributes = fileAttributeDirectory
	ns.NumberOfLinks = 1
	empty := true
	for _, fav := range favs {
		if fav.Public != fl.public {
			continue
		}
		empty = false
		ns.Name = fav.Name
		err = callback(&ns)
		if err != nil {
			return err
		}
	}
	if empty {
		return dokan.ErrObjectNameNotFound
	}
	return nil
}

func (fl *FolderList) isValidAliasTarget(ctx context.Context, nameToTry string) bool {
	return libkbfs.CheckTlfHandleOffline(ctx, nameToTry, fl.public) == nil
}

func (fl *FolderList) lockedAddChild(name string, val fileOpener) {
	fl.mu.Lock()
	fl.folders[name] = val
	fl.mu.Unlock()
}

func (fl *FolderList) updateTlfName(ctx context.Context, oldName string,
	newName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	tlf, ok := fl.folders[oldName]
	if !ok {
		return
	}

	fl.fs.log.CDebugf(ctx, "Folder name updated: %s -> %s", oldName, newName)
	delete(fl.folders, oldName)
	fl.folders[newName] = tlf
	// TODO: invalidate kernel cache for this name? (Make sure to
	// do so outside of the lock!)
}
<commit_msg>libdokan: Fix non-creation of special new folders<commit_after>// Copyright 2016 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

// +build windows

package libdokan

import (
	"sync"

	"github.com/keybase/kbfs/dokan"
	"github.com/keybase/kbfs/libkbfs"
	"golang.org/x/net/context"
)

type fileOpener interface {
	open(ctx context.Context, oc *openContext, path []string) (f dokan.File, isDir bool, err error)
	dokan.File
}

// FolderList is a node that can list all of the logged-in user's
// favorite top-level folders, on either a public or private basis.
type FolderList struct {
	emptyFile
	fs *FS
	// only accept public folders
	public bool

	mu      sync.Mutex
	folders map[string]fileOpener
}

// GetFileInformation for dokan.
func (*FolderList) GetFileInformation(*dokan.FileInfo) (*dokan.Stat, error) {
	return defaultDirectoryInformation()
}

func (fl *FolderList) reportErr(ctx context.Context,
	mode libkbfs.ErrorModeType, tlfName libkbfs.CanonicalTlfName, err error, cancelFn func()) {
	if cancelFn != nil {
		defer cancelFn()
	}
	if err == nil {
		fl.fs.log.CDebugf(ctx, "Request complete")
		return
	}

	fl.fs.config.Reporter().ReportErr(ctx, tlfName, fl.public, mode, err)
	// We just log the error as debug, rather than error, because it
	// might just indicate an expected error such as an ENOENT.
	//
	// TODO: Classify errors and escalate the logging level of the
	// important ones.
	fl.fs.log.CDebugf(ctx, err.Error())

}

// open tries to open the correct thing. Following aliases and deferring to
// Dir.open as necessary.
func (fl *FolderList) open(ctx context.Context, oc *openContext, path []string) (f dokan.File, isDir bool, err error) {
	fl.fs.log.CDebugf(ctx, "FL Lookup %#v public=%v", path, fl.public)
	if len(path) == 0 {
		return oc.returnDirNoCleanup(fl)
	}

	defer func() {
		fl.reportErr(ctx, libkbfs.ReadMode, libkbfs.CanonicalTlfName(path[0]), err, nil)
	}()

	for oc.reduceRedirectionsLeft() {
		name := path[0]

		if name == "desktop.ini" {
			fl.fs.log.CDebugf(ctx, "FL Lookup ignoring desktop.ini")
			return nil, false, dokan.ErrObjectNameNotFound
		}

		fl.mu.Lock()
		child, ok := fl.folders[name]
		fl.mu.Unlock()

		if ok {
			fl.fs.log.CDebugf(ctx, "FL Lookup recursing to child %q", name)
			return child.open(ctx, oc, path[1:])
		}

		if len(path) == 1 && isNewFolderName(name) {
			if !oc.isCreateDirectory() {
				return nil, false, dokan.ErrObjectNameNotFound
			}
			fl.fs.log.CDebugf(ctx, "FL Lookup creating EmptyFolder for Explorer")
			e := &EmptyFolder{}
			fl.lockedAddChild(name, e)
			return e, true, nil
		}

		h, err := libkbfs.ParseTlfHandle(
			ctx, fl.fs.config.KBPKI(), name, fl.public)
		fl.fs.log.CDebugf(ctx, "FL Lookup continuing -> %v,%v", h, err)
		switch err := err.(type) {
		case nil:
			// No error.
			break

		case libkbfs.TlfNameNotCanonical:
			// Only permit Aliases to targets that contain no errors.
			if !fl.isValidAliasTarget(ctx, err.NameToTry) {
				fl.fs.log.CDebugf(ctx, "FL Refusing alias to non-valid target %q", err.NameToTry)
				return nil, false, dokan.ErrObjectNameNotFound
			}

			if len(path) == 1 && oc.isOpenReparsePoint() {
				// Non-canonical name.
				n := &Alias{
					canon: err.NameToTry,
				}
				return n, true, nil
			}
			path[0] = err.NameToTry
			continue

		case libkbfs.NoSuchNameError, libkbfs.BadTLFNameError:
			return nil, false, dokan.ErrObjectNameNotFound

		default:
			// Some other error.
			return nil, false, err
		}

		fl.fs.log.CDebugf(ctx, "FL Lookup adding new child")
		child = newTLF(fl, h)
		fl.lockedAddChild(name, child)
		return child.open(ctx, oc, path[1:])
	}
	return nil, false, dokan.ErrObjectNameNotFound
}

func (fl *FolderList) forgetFolder(folderName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	delete(fl.folders, folderName)
}

// FindFiles for dokan.
func (fl *FolderList) FindFiles(fi *dokan.FileInfo, callback func(*dokan.NamedStat) error) (err error) {
	ctx, cancel := NewContextWithOpID(fl.fs, "FL FindFiles")
	defer func() { fl.fs.reportErr(ctx, libkbfs.ReadMode, err, cancel) }()

	_, _, err = fl.fs.config.KBPKI().GetCurrentUserInfo(ctx)
	isLoggedIn := err == nil

	var favs []libkbfs.Favorite
	if isLoggedIn {
		favs, err = fl.fs.config.KBFSOps().GetFavorites(ctx)
		if err != nil {
			return err
		}
	}
	var ns dokan.NamedStat
	ns.FileAttributes = fileAttributeDirectory
	ns.NumberOfLinks = 1
	empty := true
	for _, fav := range favs {
		if fav.Public != fl.public {
			continue
		}
		empty = false
		ns.Name = fav.Name
		err = callback(&ns)
		if err != nil {
			return err
		}
	}
	if empty {
		return dokan.ErrObjectNameNotFound
	}
	return nil
}

func (fl *FolderList) isValidAliasTarget(ctx context.Context, nameToTry string) bool {
	return libkbfs.CheckTlfHandleOffline(ctx, nameToTry, fl.public) == nil
}

func (fl *FolderList) lockedAddChild(name string, val fileOpener) {
	fl.mu.Lock()
	fl.folders[name] = val
	fl.mu.Unlock()
}

func (fl *FolderList) updateTlfName(ctx context.Context, oldName string,
	newName string) {
	fl.mu.Lock()
	defer fl.mu.Unlock()
	tlf, ok := fl.folders[oldName]
	if !ok {
		return
	}

	fl.fs.log.CDebugf(ctx, "Folder name updated: %s -> %s", oldName, newName)
	delete(fl.folders, oldName)
	fl.folders[newName] = tlf
	// TODO: invalidate kernel cache for this name? (Make sure to
	// do so outside of the lock!)
}
<|endoftext|>


<commit_before>package keymux

import (
	"errors"
	"hash/crc32"
	"sync"
)

var (
	ErrStopped    = errors.New("mux is stopped")
	ErrFullBuffer = errors.New("worker buffer is full")
)

type panicError struct{ reason interface{} }

func (err panicError) Error() string { return "panic occurrs" }

type Handler interface {
	Handle(key []byte, val interface{}) (interface{}, error)
}

type job struct {
	key []byte
	val interface{}
	res chan res
}

type res struct {
	val interface{}
	err error
}

func (j job) do(handler Handler) {
	defer close(j.res)
	defer func() {
		if reason := recover(); reason != nil {
			j.res <- res{nil, panicError{reason}}
		}
	}()
	val, err := handler.Handle(j.key, j.val)
	j.res <- res{val, err}
}

type Mux struct {
	mu sync.RWMutex
	wg sync.WaitGroup
	ws []chan job
}

func Start(handler Handler, workers, bufferSize int) *Mux {
	m := new(Mux)
	m.Start(handler, workers, bufferSize)
	return m
}

func (m *Mux) Start(handler Handler, workers, bufferSize int) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.ws != nil {
		panic("mux is already started")
	}
	if workers <= 0 {
		panic("number of workers must be positive")
	}
	if bufferSize < 0 {
		panic("worker buffer size must not be negative")
	}
	m.wg.Add(workers)
	m.ws = make([]chan job, workers)
	for i := range m.ws {
		m.ws[i] = make(chan job, bufferSize)
		go func(jobs <-chan job) {
			defer m.wg.Done()
			for j := range jobs {
				j.do(handler)
			}
		}(m.ws[i])
	}
}

func (m *Mux) Stop() {
	m.mu.Lock()
	defer m.mu.Unlock()
	for _, jobs := range m.ws {
		close(jobs)
	}
	m.ws = nil
	m.wg.Wait()
}

func (m *Mux) handle(crc uint32, j job) error {
	m.mu.RLock()
	defer m.mu.RUnlock()
	if m.ws == nil {
		return ErrStopped
	}
	w := crc % uint32(len(m.ws))
	select {
	case m.ws[w] <- j:
		return nil
	default:
		return ErrFullBuffer
	}
}

func (m *Mux) Handle(key []byte, val interface{}) (interface{}, error) {
	j := job{key, val, make(chan res)}
	err := m.handle(crc32.ChecksumIEEE(key), j)
	if err != nil {
		return nil, err
	}
	r := <-j.res
	if p, ok := r.err.(panicError); ok {
		panic(p.reason)
	}
	return r.val, r.err
}
<commit_msg>Add Fixed and Urgent handler wrappers<commit_after>package keymux

import (
	"errors"
	"hash/crc32"
	"sync"
)

var (
	ErrStopped    = errors.New("mux is stopped")
	ErrFullBuffer = errors.New("worker buffer is full")
)

type panicError struct{ reason interface{} }

func (err panicError) Error() string { return "panic occurrs" }

type Handler interface {
	Handle(key []byte, val interface{}) (interface{}, error)
}

type FixedHandler interface {
	Handle(val interface{}) (interface{}, error)
}

type job struct {
	key []byte
	val interface{}
	res chan res
}

type res struct {
	val interface{}
	err error
}

func (j job) do(handler Handler) {
	defer close(j.res)
	defer func() {
		if reason := recover(); reason != nil {
			j.res <- res{nil, panicError{reason}}
		}
	}()
	val, err := handler.Handle(j.key, j.val)
	j.res <- res{val, err}
}

type Mux struct {
	mu sync.RWMutex
	wg sync.WaitGroup
	ws []chan job
}

func Start(handler Handler, workers, bufferSize int) *Mux {
	m := new(Mux)
	m.Start(handler, workers, bufferSize)
	return m
}

func (m *Mux) Start(handler Handler, workers, bufferSize int) {
	m.mu.Lock()
	defer m.mu.Unlock()
	if m.ws != nil {
		panic("mux is already started")
	}
	if workers <= 0 {
		panic("number of workers must be positive")
	}
	if bufferSize < 0 {
		panic("worker buffer size must not be negative")
	}
	m.wg.Add(workers)
	m.ws = make([]chan job, workers)
	for i := range m.ws {
		m.ws[i] = make(chan job, bufferSize)
		go func(jobs <-chan job) {
			defer m.wg.Done()
			for j := range jobs {
				j.do(handler)
			}
		}(m.ws[i])
	}
}

func (m *Mux) Stop() {
	m.mu.Lock()
	defer m.mu.Unlock()
	for _, jobs := range m.ws {
		close(jobs)
	}
	m.ws = nil
	m.wg.Wait()
}

func (m *Mux) handle(crc uint32, j job) error {
	m.mu.RLock()
	defer m.mu.RUnlock()
	if m.ws == nil {
		return ErrStopped
	}
	w := crc % uint32(len(m.ws))
	select {
	case m.ws[w] <- j:
		return nil
	default:
		return ErrFullBuffer
	}
}

func (m *Mux) Handle(key []byte, val interface{}) (interface{}, error) {
	j := job{key, val, make(chan res)}
	err := m.handle(crc32.ChecksumIEEE(key), j)
	if err != nil {
		return nil, err
	}
	r := <-j.res
	if p, ok := r.err.(panicError); ok {
		panic(p.reason)
	}
	return r.val, r.err
}

type fixed struct {
	handler Handler
	key     []byte
}

func Fixed(handler Handler, key []byte) FixedHandler {
	return fixed{handler: handler, key: key}
}

func (f fixed) Handle(val interface{}) (interface{}, error) {
	return f.handler.Handle(f.key, val)
}

type urgent struct {
	handler Handler
}

func Urgent(handler Handler) Handler {
	return urgent{handler: handler}
}

func (u urgent) Handle(key []byte, val interface{}) (interface{}, error) {
	for {
		if val, err := u.handler.Handle(key, val); err != ErrFullBuffer {
			return val, err
		}
	}
}
<|endoftext|>


<commit_before>package readraptor

import (
	"time"

	"github.com/coopernurse/gorp"
	"github.com/technoweenie/grohl"
)

type ReadReceipt struct {
	Id         int64     `db:"id"`
	Created    time.Time `db:"created_at"`
	LastReadAt time.Time `db:"last_read_at"`
	ArticleId  int64     `db:"article_id"`
	ReaderId   int64     `db:"reader_id"`
}

func TrackReadReceipt(dbmap *gorp.DbMap, account *Account, key, reader string) error {
	cid, err := InsertArticle(dbmap, account.Id, key)
	if err != nil {
		return err
	}

	vid, err := InsertReader(dbmap, account.Id, reader)
	if err != nil {
		return err
	}

	_, err = UpsertReadReceipt(dbmap, cid, vid)
	if err != nil {
		return err
	}

	grohl.Log(grohl.Data{
		"account": account.Id,
		"reader":  reader,
		"article": key,
		"track":   "read",
	})

	return nil
}

func UpsertReadReceipt(dbmap *gorp.DbMap, articleId, readerId int64) (int64, error) {
	id, err := dbmap.SelectNullInt(`
        update read_receipts set last_read_at = $1 where article_id=$2 and reader_id=$3 returning id;
    `, time.Now().UTC(),
		articleId,
		readerId,
	)
	if err != nil {
		return -1, err
	}
	if id.Valid {
		return id.Int64, nil
	}

	id, err = dbmap.SelectNullInt(`
        with s as (
            select id from read_receipts where article_id = $1 and reader_id = $2
        ), i as (
            insert into read_receipts ("article_id", "reader_id", "created_at", "last_read_at")
            select $1, $2, $3, $3
            where not exists (select 1 from s)
            returning id
        )
        select id from i union all select id from s;
    `, articleId,
		readerId,
		time.Now().UTC(),
	)
	if err != nil {
		return -1, err
	}

	iid, err := id.Value()

	return iid.(int64), err
}

func UnreadArticles(dbmap *gorp.DbMap, readerId int64) (keys []Article, err error) {
	_, err = dbmap.Select(&keys, `
        select articles.*, read_receipts.created_at as first_read_at, last_read_at from
            (select article_id from expected_readers where reader_id = $1
                except all
             select article_id from read_receipts
						   inner join articles on articles.id = read_receipts.article_id
						 where reader_id = $1 and articles.updated_at < read_receipts.last_read_at) unread_articles
        inner join articles on articles.id = unread_articles.article_id
				 left join read_receipts on read_receipts.article_id = unread_articles.article_id;`, readerId)

	return
}

func UnreadArticlesMarkRead(dbmap *gorp.DbMap, readerId int64) (keys []string, err error) {
	t, err := dbmap.Begin()
	if err != nil {
		return
	}

	articles, err := UnreadArticles(dbmap, readerId)
	if err != nil {
		return
	}

	keys = make([]string, 0)
	for _, a := range articles {
		keys = append(keys, a.Key)
	}

	_, err = dbmap.Select(&keys, `
        select articles.* from
            (select article_id from expected_readers where reader_id = $1
                except all
             select article_id from read_receipts where reader_id = $1) unread_articles
        inner join articles on articles.id = unread_articles.article_id;`, readerId)

	// Mark articles as read
	for _, ci := range articles {
		_, err = UpsertReadReceipt(dbmap, ci.Id, readerId)
		if err != nil {
			return
		}
	}

	err = t.Commit()

	return
}
<commit_msg>Select distinct to prevent duplicates<commit_after>package readraptor

import (
	"time"

	"github.com/coopernurse/gorp"
	"github.com/technoweenie/grohl"
)

type ReadReceipt struct {
	Id         int64     `db:"id"`
	Created    time.Time `db:"created_at"`
	LastReadAt time.Time `db:"last_read_at"`
	ArticleId  int64     `db:"article_id"`
	ReaderId   int64     `db:"reader_id"`
}

func TrackReadReceipt(dbmap *gorp.DbMap, account *Account, key, reader string) error {
	cid, err := InsertArticle(dbmap, account.Id, key)
	if err != nil {
		return err
	}

	vid, err := InsertReader(dbmap, account.Id, reader)
	if err != nil {
		return err
	}

	_, err = UpsertReadReceipt(dbmap, cid, vid)
	if err != nil {
		return err
	}

	grohl.Log(grohl.Data{
		"account": account.Id,
		"reader":  reader,
		"article": key,
		"track":   "read",
	})

	return nil
}

func UpsertReadReceipt(dbmap *gorp.DbMap, articleId, readerId int64) (int64, error) {
	id, err := dbmap.SelectNullInt(`
        update read_receipts set last_read_at = $1 where article_id=$2 and reader_id=$3 returning id;
    `, time.Now().UTC(),
		articleId,
		readerId,
	)
	if err != nil {
		return -1, err
	}
	if id.Valid {
		return id.Int64, nil
	}

	id, err = dbmap.SelectNullInt(`
        with s as (
            select id from read_receipts where article_id = $1 and reader_id = $2
        ), i as (
            insert into read_receipts ("article_id", "reader_id", "created_at", "last_read_at")
            select $1, $2, $3, $3
            where not exists (select 1 from s)
            returning id
        )
        select id from i union all select id from s;
    `, articleId,
		readerId,
		time.Now().UTC(),
	)
	if err != nil {
		return -1, err
	}

	iid, err := id.Value()

	return iid.(int64), err
}

func UnreadArticles(dbmap *gorp.DbMap, readerId int64) (keys []Article, err error) {
	_, err = dbmap.Select(&keys, `
        select distinct articles.*, read_receipts.created_at as first_read_at, last_read_at from
            (select article_id from expected_readers where reader_id = $1
                except all
             select article_id from read_receipts
						   inner join articles on articles.id = read_receipts.article_id
						 where reader_id = $1 and articles.updated_at < read_receipts.last_read_at) unread_articles
        inner join articles on articles.id = unread_articles.article_id
				 left join read_receipts on read_receipts.article_id = unread_articles.article_id;`, readerId)

	return
}

func UnreadArticlesMarkRead(dbmap *gorp.DbMap, readerId int64) (keys []string, err error) {
	t, err := dbmap.Begin()
	if err != nil {
		return
	}

	articles, err := UnreadArticles(dbmap, readerId)
	if err != nil {
		return
	}

	keys = make([]string, 0)
	for _, a := range articles {
		keys = append(keys, a.Key)
	}

	_, err = dbmap.Select(&keys, `
        select articles.* from
            (select article_id from expected_readers where reader_id = $1
                except all
             select article_id from read_receipts where reader_id = $1) unread_articles
        inner join articles on articles.id = unread_articles.article_id;`, readerId)

	// Mark articles as read
	for _, ci := range articles {
		_, err = UpsertReadReceipt(dbmap, ci.Id, readerId)
		if err != nil {
			return
		}
	}

	err = t.Commit()

	return
}
<|endoftext|>


<commit_before><commit_msg>Writeファイルの処理を変更.<commit_after><|endoftext|>


<commit_before>package def

import (
	"encoding/json"
	"fmt"
	"sort"
	"time"

	"github.com/ugorji/go/codec"
)

//
// RunRecord
//

var _ codec.Selfer = &_assertHelper

func (rr RunRecord) CodecEncodeSelf(c *codec.Encoder) {
	// Copy pretty much the entire struct over to an anonymous one,
	//  as a way of saying "do all the normal things you would with tags";
	//  we just want to inject an Opinion on this one field.
	//
	// To be clear: this is terrible.
	//  My kingdom for a next generation of serialization/structmapping
	//  tools which actually let me pick out the *field* without this
	//  copypasta trainwreck of misplaced ambitions.
	c.MustEncode(struct {
		HID        string      `json:"HID,omitempty"`
		UID        RunID       `json:"UID"`
		Date       time.Time   `json:"when"`
		FormulaHID string      `json:"formulaHID,omitempty"`
		Results    ResultGroup `json:"results"`
		Failure    error       `json:"failure,omitempty"`
	}{
		HID:        rr.HID,
		UID:        rr.UID,
		Date:       rr.Date,
		FormulaHID: rr.FormulaHID,
		Results:    rr.Results,
		Failure: func() error {
			if rr.Failure == nil {
				return nil
			}
			return runRecordFailureEnvelope{
				failureTypeToString(rr.Failure),
				rr.Failure,
			}
		}(),
	})
}

func (rr *RunRecord) CodecDecodeSelf(c *codec.Decoder) {
	failureEnvelope := &runRecordFailureEnvelope{
		Detail: json.RawMessage{},
	}
	rusrs := struct {
		HID        string      `json:"HID,omitempty"`
		UID        RunID       `json:"UID"`
		Date       time.Time   `json:"when"`
		FormulaHID string      `json:"formulaHID,omitempty"`
		Results    ResultGroup `json:"results"`
		Failure    error       `json:"failure,omitempty"`
	}{
		Failure: failureEnvelope,
	}
	c.MustDecode(&rusrs)
	rr.HID = rusrs.HID
	rr.UID = rusrs.UID
	rr.Date = rusrs.Date
	rr.FormulaHID = rusrs.FormulaHID
	rr.Results = rusrs.Results
	if failureEnvelope.Type != "" {
		realFailure := stringToBlankFailure(failureEnvelope.Type)
		// I give up.  The following is broken:
		//  we use a json serializer here.  This will break for CBOR.
		//  I tried many other things.
		//  See the commit history immediately preceeding this area.
		//  The fix for this is *changing codec libraries entirely*.
		err := json.Unmarshal(failureEnvelope.Detail.(json.RawMessage), realFailure)
		if err != nil {
			panic(&ErrUnmarshalling{
				Msg: fmt.Sprintf("cannot unmarshal error type: %s", err),
			})
		}
		rr.Failure = realFailure
	}
}

type runRecordFailureEnvelope struct {
	Type   string      `json:"type"`
	Detail interface{} `json:"detail"`
}

func (runRecordFailureEnvelope) Error() string { return "" }

func failureTypeToString(e error) string {
	switch e.(type) {
	case *ErrConfigParsing:
		return "ErrConfigParsing"
	case *ErrConfigValidation:
		return "ErrConfigValidation"
	case *ErrWarehouseUnavailable:
		return "ErrWarehouseUnavailable"
	case *ErrWarehouseProblem:
		return "ErrWarehouseProblem"
	case *ErrWareDNE:
		return "ErrWareDNE"
	case *ErrWareCorrupt:
		return "ErrWareCorrupt"
	default:
		panic(fmt.Errorf("Internal Error type %T not suitable for API.\n\tFull error: %s", e, e))
	}
}

func stringToBlankFailure(typ string) error {
	switch typ {
	case "ErrConfigParsing":
		return &ErrConfigParsing{}
	case "ErrConfigValidation":
		return &ErrConfigValidation{}
	case "ErrWarehouseUnavailable":
		return &ErrWarehouseUnavailable{}
	case "ErrWarehouseProblem":
		return &ErrWarehouseProblem{}
	case "ErrWareDNE":
		return &ErrWareDNE{}
	case "ErrWareCorrupt":
		return &ErrWareCorrupt{}
	default:
		panic(&ErrUnmarshalling{
			Msg: fmt.Sprintf("cannot unmarshal error type: %q is not a known type", typ),
		})
	}
}

//
// ResultGroup
//

var _ codec.Selfer = &ResultGroup{}

func (rg ResultGroup) CodecEncodeSelf(c *codec.Encoder) {
	c.Encode(rg.asMappySlice())
}

func (rg *ResultGroup) CodecDecodeSelf(c *codec.Decoder) {
	// I'd love to just punt to the defaults, but the `Selfer` interface doesn't come in half.
	// SO here's a ridiculous indirection to prance around infinite recursion.
	c.MustDecode((*map[string]*Result)(rg))
}

func (mp ResultGroup) asMappySlice() codec.MapBySlice {
	keys := make([]string, len(mp))
	var i int
	for k := range mp {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	val := make(mappySlice, len(mp)*2)
	i = 0
	for _, k := range keys {
		val[i] = k
		i++
		val[i] = mp[k]
		i++
	}
	return val
}
<commit_msg>Fix erroneous ommisson of ErrHashMismatch type from the polymorphism table for the RunRecord.Failure field.<commit_after>package def

import (
	"encoding/json"
	"fmt"
	"sort"
	"time"

	"github.com/ugorji/go/codec"
)

//
// RunRecord
//

var _ codec.Selfer = &_assertHelper

func (rr RunRecord) CodecEncodeSelf(c *codec.Encoder) {
	// Copy pretty much the entire struct over to an anonymous one,
	//  as a way of saying "do all the normal things you would with tags";
	//  we just want to inject an Opinion on this one field.
	//
	// To be clear: this is terrible.
	//  My kingdom for a next generation of serialization/structmapping
	//  tools which actually let me pick out the *field* without this
	//  copypasta trainwreck of misplaced ambitions.
	c.MustEncode(struct {
		HID        string      `json:"HID,omitempty"`
		UID        RunID       `json:"UID"`
		Date       time.Time   `json:"when"`
		FormulaHID string      `json:"formulaHID,omitempty"`
		Results    ResultGroup `json:"results"`
		Failure    error       `json:"failure,omitempty"`
	}{
		HID:        rr.HID,
		UID:        rr.UID,
		Date:       rr.Date,
		FormulaHID: rr.FormulaHID,
		Results:    rr.Results,
		Failure: func() error {
			if rr.Failure == nil {
				return nil
			}
			return runRecordFailureEnvelope{
				failureTypeToString(rr.Failure),
				rr.Failure,
			}
		}(),
	})
}

func (rr *RunRecord) CodecDecodeSelf(c *codec.Decoder) {
	failureEnvelope := &runRecordFailureEnvelope{
		Detail: json.RawMessage{},
	}
	rusrs := struct {
		HID        string      `json:"HID,omitempty"`
		UID        RunID       `json:"UID"`
		Date       time.Time   `json:"when"`
		FormulaHID string      `json:"formulaHID,omitempty"`
		Results    ResultGroup `json:"results"`
		Failure    error       `json:"failure,omitempty"`
	}{
		Failure: failureEnvelope,
	}
	c.MustDecode(&rusrs)
	rr.HID = rusrs.HID
	rr.UID = rusrs.UID
	rr.Date = rusrs.Date
	rr.FormulaHID = rusrs.FormulaHID
	rr.Results = rusrs.Results
	if failureEnvelope.Type != "" {
		realFailure := stringToBlankFailure(failureEnvelope.Type)
		// I give up.  The following is broken:
		//  we use a json serializer here.  This will break for CBOR.
		//  I tried many other things.
		//  See the commit history immediately preceeding this area.
		//  The fix for this is *changing codec libraries entirely*.
		err := json.Unmarshal(failureEnvelope.Detail.(json.RawMessage), realFailure)
		if err != nil {
			panic(&ErrUnmarshalling{
				Msg: fmt.Sprintf("cannot unmarshal error type: %s", err),
			})
		}
		rr.Failure = realFailure
	}
}

type runRecordFailureEnvelope struct {
	Type   string      `json:"type"`
	Detail interface{} `json:"detail"`
}

func (runRecordFailureEnvelope) Error() string { return "" }

func failureTypeToString(e error) string {
	switch e.(type) {
	case *ErrConfigParsing:
		return "ErrConfigParsing"
	case *ErrConfigValidation:
		return "ErrConfigValidation"
	case *ErrWarehouseUnavailable:
		return "ErrWarehouseUnavailable"
	case *ErrWarehouseProblem:
		return "ErrWarehouseProblem"
	case *ErrWareDNE:
		return "ErrWareDNE"
	case *ErrHashMismatch:
		return "ErrHashMismatch"
	case *ErrWareCorrupt:
		return "ErrWareCorrupt"
	default:
		panic(fmt.Errorf("Internal Error type %T not suitable for API.\n\tFull error: %s", e, e))
	}
}

func stringToBlankFailure(typ string) error {
	switch typ {
	case "ErrConfigParsing":
		return &ErrConfigParsing{}
	case "ErrConfigValidation":
		return &ErrConfigValidation{}
	case "ErrWarehouseUnavailable":
		return &ErrWarehouseUnavailable{}
	case "ErrWarehouseProblem":
		return &ErrWarehouseProblem{}
	case "ErrWareDNE":
		return &ErrWareDNE{}
	case "ErrHashMismatch":
		return &ErrHashMismatch{}
	case "ErrWareCorrupt":
		return &ErrWareCorrupt{}
	default:
		panic(&ErrUnmarshalling{
			Msg: fmt.Sprintf("cannot unmarshal error type: %q is not a known type", typ),
		})
	}
}

//
// ResultGroup
//

var _ codec.Selfer = &ResultGroup{}

func (rg ResultGroup) CodecEncodeSelf(c *codec.Encoder) {
	c.Encode(rg.asMappySlice())
}

func (rg *ResultGroup) CodecDecodeSelf(c *codec.Decoder) {
	// I'd love to just punt to the defaults, but the `Selfer` interface doesn't come in half.
	// SO here's a ridiculous indirection to prance around infinite recursion.
	c.MustDecode((*map[string]*Result)(rg))
}

func (mp ResultGroup) asMappySlice() codec.MapBySlice {
	keys := make([]string, len(mp))
	var i int
	for k := range mp {
		keys[i] = k
		i++
	}
	sort.Strings(keys)
	val := make(mappySlice, len(mp)*2)
	i = 0
	for _, k := range keys {
		val[i] = k
		i++
		val[i] = mp[k]
		i++
	}
	return val
}
<|endoftext|>


<commit_before>// run_chromium_perf_swarming is an application that is meant to be run on
// a swarming slave. It runs the specified benchmark over CT's webpage
// archives and uploads results to chromeperf.appspot.com
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"time"

	"github.com/skia-dev/glog"

	"strings"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	workerNum          = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	chromiumBuild      = flag.String("chromium_build", "", "The chromium build dir to use.")
	benchmarkName      = flag.String("benchmark_name", "", "The telemetry benchmark to run on this worker.")
	benchmarkExtraArgs = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	browserExtraArgs   = flag.String("browser_extra_args", "--disable-setuid-sandbox --enable-threaded-compositing --enable-impl-side-painting", "The extra arguments that are passed to the browser while running the benchmark.")
	// TODO(rmistry): Make this 3 instead of 1? runs will take longer.
	repeatBenchmark      = flag.Int("repeat_benchmark", 1, "The number of times the benchmark should be repeated. For skpicture_printer benchmark this value is always 1.")
	targetPlatform       = flag.String("target_platform", util.PLATFORM_LINUX, "The platform the benchmark will run on (Android / Linux).")
	telemetryBinariesDir = flag.String("telemetry_binaries_dir", "", "The directory that contains the telemetry binaries.")
	pageSetsDir          = flag.String("page_sets_dir", "", "The directory that contains the CT page sets.")
	// Below flags are used to upload data to chromeperf.
	buildbotMaster  = flag.String("master", "", "The name of the buildbot master the builder is running on.")
	buildbotBuilder = flag.String("builder", "", "The name of the buildbot builder that triggered this script.")
	gitHash         = flag.String("git_hash", "", "The git hash the build was triggered at.")
)

func main() {
	defer common.LogPanic()
	worker_common.Init()

	defer util.TimeTrack(time.Now(), "Running Chromium Perf on Swarming")
	defer glog.Flush()

	// Validate required arguments.
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *benchmarkName == "" {
		glog.Error("Must specify --benchmark_name")
		return
	}
	if *telemetryBinariesDir == "" {
		glog.Error("Must specify --telemetry_binaries_dir")
		return
	}
	if *pageSetsDir == "" {
		glog.Error("Must specify --page_sets_dir")
		return
	}
	if *buildbotMaster == "" {
		glog.Error("Must specify --master")
		return
	}
	if *buildbotBuilder == "" {
		glog.Error("Must specify --builder")
		return
	}
	if *gitHash == "" {
		glog.Error("Must specify --git_hash")
		return
	}
	chromiumBinary := filepath.Join(*chromiumBuild, util.BINARY_CHROME)

	// Establish output paths.
	localOutputDir := util.BenchmarkRunsDir
	skutil.MkdirAll(localOutputDir, 0700)

	fileInfos, err := ioutil.ReadDir(*pageSetsDir)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", *pageSetsDir, err)
		return
	}

	for _, fileInfo := range fileInfos {
		if fileInfo.IsDir() {
			continue
		}
		if err := runBenchmark(fileInfo.Name(), *pageSetsDir, localOutputDir, *chromiumBuild, chromiumBinary, *browserExtraArgs); err != nil {
			glog.Errorf("Error while running benchmark: %s", err)
			return
		}
	}

	// Convert output to dashboard JSON v1 in order to upload to chromeperf.
	// More information is in http://www.chromium.org/developers/speed-infra/performance-dashboard/sending-data-to-the-performance-dashboard
	client := skutil.NewTimeoutClient()
	outputFileInfos, err := ioutil.ReadDir(localOutputDir)
	if err != nil {
		glog.Errorf("Unable to read %s: %s", localOutputDir, err)
		return
	}
	for _, fileInfo := range outputFileInfos {
		if !fileInfo.IsDir() {
			continue
		}
		resultsFile := filepath.Join(localOutputDir, fileInfo.Name(), "results-chart.json")
		if err := uploadResultsToPerfDashboard(resultsFile, client); err != nil {
			glog.Errorf("Could not upload to perf dashboard: %s", err)
			continue
		}
	}
}

func runBenchmark(fileInfoName, pathToPagesets, localOutputDir, chromiumBuildName, chromiumBinary, browserExtraArgs string) error {
	pagesetBaseName := filepath.Base(fileInfoName)
	if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
		// Ignore timestamp files and .pyc files.
		return nil
	}

	// Read the pageset.
	pagesetName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
	pagesetPath := filepath.Join(pathToPagesets, fileInfoName)
	decodedPageset, err := util.ReadPageset(pagesetPath)
	if err != nil {
		return fmt.Errorf("Could not read %s: %s", pagesetPath, err)
	}

	glog.Infof("===== Processing %s =====", pagesetPath)
	benchmark, present := util.BenchmarksToTelemetryName[*benchmarkName]
	if !present {
		// If it is custom benchmark use the entered benchmark name.
		benchmark = *benchmarkName
	}
	args := []string{
		filepath.Join(*telemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
		benchmark,
		"--also-run-disabled-tests",
		"--user-agent=" + decodedPageset.UserAgent,
		"--urls-list=" + decodedPageset.UrlsList,
		"--archive-data-file=" + filepath.Join(*pageSetsDir, decodedPageset.ArchiveDataFile),
		// Output in chartjson. Needed for uploading to performance dashboard.
		// Documentation is here: http://www.chromium.org/developers/speed-infra/performance-dashboard/sending-data-to-the-performance-dashboard
		"--output-format=chartjson",
		// Upload traces.
		"--upload-results",
		"--upload-bucket=output",
	}

	// Need to capture output for all benchmarks.
	outputDirArgValue := filepath.Join(localOutputDir, pagesetName)
	args = append(args, "--output-dir="+outputDirArgValue)
	// Figure out which browser should be used.
	if *targetPlatform == util.PLATFORM_ANDROID {
		if err := util.InstallChromeAPK(chromiumBuildName); err != nil {
			return fmt.Errorf("Error while installing APK: %s", err)
		}
		args = append(args, "--browser=android-chromium")
	} else {
		args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		args = append(args, strings.Split(*benchmarkExtraArgs, " ")...)
	}
	// Add the number of times to repeat.
	args = append(args, fmt.Sprintf("--page-repeat=%d", *repeatBenchmark))
	// Add browserArgs if not empty to args.
	if browserExtraArgs != "" {
		args = append(args, "--extra-browser-args="+browserExtraArgs)
	}
	// Set the PYTHONPATH to the pagesets and the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", *telemetryBinariesDir),
		"DISPLAY=:0",
	}
	timeoutSecs := 2 * 60 // 2 mins timeout
	if err := util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil); err != nil {
		glog.Errorf("Run benchmark command failed with: %s", err)
	}
	return nil
}

// The master name used for the dashboard is a CamelCase name not the
// canonical master name with dots.
func getCamelCaseMasterName(master string) string {
	// create it from golang playground
	camelCaseMaster := ""
	tokens := strings.Split(master, ".")
	for _, token := range tokens {
		camelCaseMaster += strings.Title(token)
	}
	return camelCaseMaster
}

// JSON decodes the provided resultsFile and uploads results to chromeperf.appspot.com
func uploadResultsToPerfDashboard(resultsFile string, client *http.Client) error {
	jsonFile, err := os.Open(resultsFile)
	defer skutil.Close(jsonFile)
	if err != nil {
		return fmt.Errorf("Could not open %s: %s", resultsFile, err)
	}
	// Read the JSON and convert to dashboard JSON v1.
	var chartData interface{}
	if err := json.NewDecoder(jsonFile).Decode(&chartData); err != nil {
		return fmt.Errorf("Could not parse %s: %s", resultsFile, err)
	}
	// TODO(rmistry): Populate the below with data that can be monitored.
	versions := map[string]string{
		"chromium": *gitHash,
	}
	supplemental := map[string]string{}
	// Create a custom dictionary and convert it to JSON.
	dashboardFormat := map[string]interface{}{
		"master":       getCamelCaseMasterName(*buildbotMaster),
		"bot":          *buildbotBuilder,
		"chart_data":   chartData,
		"point_id":     time.Now().Unix(),
		"versions":     versions,
		"supplemental": supplemental,
	}
	marshalledData, err := json.Marshal(dashboardFormat)
	if err != nil {
		return fmt.Errorf("Could not create dashboard JSON for %s: %s", resultsFile, err)
	}

	// Post the above to https://chromeperf.appspot.com/add_point with one parameter called data.
	postData := url.Values{}
	postData.Set("data", string(marshalledData))
	req, err := http.NewRequest("POST", "https://chromeperf.appspot.com/add_point", strings.NewReader(postData.Encode()))
	if err != nil {
		return fmt.Errorf("Could not create HTTP request for %s: %s", resultsFile, err)
	}
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("Could not post to chromeperf for %s: %s", resultsFile, err)
	}
	defer skutil.Close(resp.Body)
	if resp.StatusCode != 200 {
		return fmt.Errorf("Could not post to chromeperf for %s, response status code was %d", resultsFile, resp.StatusCode)
	}
	glog.Infof("Successfully uploaded the following to chromeperf: %s", string(marshalledData))
	return nil
}
<commit_msg>[CT+Perf] Do not upload to chromeperf from run_chromium_perf_swarming. Instead gather all results in the recipe and upload it in one go to chromeperf.<commit_after>// run_chromium_perf_swarming is an application that is meant to be run on
// a swarming slave. It runs the specified benchmark over CT's webpage
// archives and uploads results to chromeperf.appspot.com
package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"time"

	"github.com/skia-dev/glog"

	"strings"

	"go.skia.org/infra/ct/go/util"
	"go.skia.org/infra/ct/go/worker_scripts/worker_common"
	"go.skia.org/infra/go/common"
	skutil "go.skia.org/infra/go/util"
)

var (
	workerNum            = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	chromiumBuild        = flag.String("chromium_build", "", "The chromium build dir to use.")
	benchmarkName        = flag.String("benchmark_name", "", "The telemetry benchmark to run on this worker.")
	benchmarkExtraArgs   = flag.String("benchmark_extra_args", "", "The extra arguments that are passed to the specified benchmark.")
	browserExtraArgs     = flag.String("browser_extra_args", "--disable-setuid-sandbox --enable-threaded-compositing --enable-impl-side-painting", "The extra arguments that are passed to the browser while running the benchmark.")
	repeatBenchmark      = flag.Int("repeat_benchmark", 2, "The number of times the benchmark should be repeated. For skpicture_printer benchmark this value is always 1.")
	targetPlatform       = flag.String("target_platform", util.PLATFORM_LINUX, "The platform the benchmark will run on (Android / Linux).")
	telemetryBinariesDir = flag.String("telemetry_binaries_dir", "", "The directory that contains the telemetry binaries.")
	pageSetsDir          = flag.String("page_sets_dir", "", "The directory that contains the CT page sets.")
	outDir               = flag.String("out_dir", "", "The directory to send results to.")
)

func main() {
	defer common.LogPanic()
	worker_common.Init()

	defer util.TimeTrack(time.Now(), "Running Chromium Perf on Swarming")
	defer glog.Flush()

	// Validate required arguments.
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *benchmarkName == "" {
		glog.Error("Must specify --benchmark_name")
		return
	}
	if *telemetryBinariesDir == "" {
		glog.Error("Must specify --telemetry_binaries_dir")
		return
	}
	if *pageSetsDir == "" {
		glog.Error("Must specify --page_sets_dir")
		return
	}
	if *outDir == "" {
		glog.Error("Must specify --out_dir")
		return
	}
	chromiumBinary := filepath.Join(*chromiumBuild, util.BINARY_CHROME)

	// Establish output paths.
	localOutputDir := util.BenchmarkRunsDir
	skutil.MkdirAll(localOutputDir, 0700)

	fileInfos, err := ioutil.ReadDir(*pageSetsDir)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", *pageSetsDir, err)
		return
	}

	for _, fileInfo := range fileInfos {
		if fileInfo.IsDir() {
			continue
		}
		if err := runBenchmark(fileInfo.Name(), *pageSetsDir, localOutputDir, *chromiumBuild, chromiumBinary, *browserExtraArgs); err != nil {
			glog.Errorf("Error while running benchmark: %s", err)
			return
		}
	}

	// Convert output to dashboard JSON v1 in order to upload to chromeperf.
	// More information is in http://www.chromium.org/developers/speed-infra/performance-dashboard/sending-data-to-the-performance-dashboard
	outputFileInfos, err := ioutil.ReadDir(localOutputDir)
	if err != nil {
		glog.Errorf("Unable to read %s: %s", localOutputDir, err)
		return
	}
	for _, fileInfo := range outputFileInfos {
		if !fileInfo.IsDir() {
			continue
		}
		resultsFile := filepath.Join(localOutputDir, fileInfo.Name(), "results-chart.json")
		outputFile := filepath.Join(*outDir, fmt.Sprintf("%s-results.json", fileInfo.Name()))
		if err := os.Rename(resultsFile, outputFile); err != nil {
			glog.Errorf("Could not rename %s to %s: %s", resultsFile, outputFile, err)
			continue
		}
	}
}

func runBenchmark(fileInfoName, pathToPagesets, localOutputDir, chromiumBuildName, chromiumBinary, browserExtraArgs string) error {
	pagesetBaseName := filepath.Base(fileInfoName)
	if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
		// Ignore timestamp files and .pyc files.
		return nil
	}

	// Read the pageset.
	pagesetName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
	pagesetPath := filepath.Join(pathToPagesets, fileInfoName)
	decodedPageset, err := util.ReadPageset(pagesetPath)
	if err != nil {
		return fmt.Errorf("Could not read %s: %s", pagesetPath, err)
	}

	glog.Infof("===== Processing %s =====", pagesetPath)
	benchmark, present := util.BenchmarksToTelemetryName[*benchmarkName]
	if !present {
		// If it is custom benchmark use the entered benchmark name.
		benchmark = *benchmarkName
	}
	args := []string{
		filepath.Join(*telemetryBinariesDir, util.BINARY_RUN_BENCHMARK),
		benchmark,
		"--also-run-disabled-tests",
		"--user-agent=" + decodedPageset.UserAgent,
		"--urls-list=" + decodedPageset.UrlsList,
		"--archive-data-file=" + filepath.Join(*pageSetsDir, decodedPageset.ArchiveDataFile),
		// Output in chartjson. Needed for uploading to performance dashboard.
		// Documentation is here: http://www.chromium.org/developers/speed-infra/performance-dashboard/sending-data-to-the-performance-dashboard
		"--output-format=chartjson",
		// Upload traces.
		"--upload-results",
		"--upload-bucket=output",
	}

	// Need to capture output for all benchmarks.
	outputDirArgValue := filepath.Join(localOutputDir, pagesetName)
	args = append(args, "--output-dir="+outputDirArgValue)
	// Figure out which browser should be used.
	if *targetPlatform == util.PLATFORM_ANDROID {
		if err := util.InstallChromeAPK(chromiumBuildName); err != nil {
			return fmt.Errorf("Error while installing APK: %s", err)
		}
		args = append(args, "--browser=android-chromium")
	} else {
		args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
	}
	// Split benchmark args if not empty and append to args.
	if *benchmarkExtraArgs != "" {
		args = append(args, strings.Split(*benchmarkExtraArgs, " ")...)
	}
	// Add the number of times to repeat.
	args = append(args, fmt.Sprintf("--page-repeat=%d", *repeatBenchmark))
	// Add browserArgs if not empty to args.
	if browserExtraArgs != "" {
		args = append(args, "--extra-browser-args="+browserExtraArgs)
	}
	// Set the PYTHONPATH to the pagesets and the telemetry dirs.
	env := []string{
		fmt.Sprintf("PYTHONPATH=%s:$PYTHONPATH", *telemetryBinariesDir),
		"DISPLAY=:0",
	}
	timeoutSecs := 2 * 60 // 2 mins timeout
	if err := util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil); err != nil {
		glog.Errorf("Run benchmark command failed with: %s", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package exposer

import (
	"encoding/json"
	"errors"
	"io"
	"net"
	"sync"

	"github.com/inconshreveable/muxado"
)

type HandshakeHandleFunc func(proto *Protocal, cmd string, details []byte) error
type Protocal struct {
	conn             net.Conn
	isHandshakeDone  bool
	handshakeDecoder *json.Decoder
	eventbus         chan HandshakeIncoming
	done             chan struct{}

	setErrOnce *sync.Once
	err        error

	// handle handshake
	mutex_On *sync.Mutex
	On       HandshakeHandleFunc
}

func NewProtocal(conn net.Conn) *Protocal {
	return &Protocal{
		conn:             conn,
		isHandshakeDone:  false,
		handshakeDecoder: json.NewDecoder(conn),
		eventbus:         make(chan HandshakeIncoming),
		done:             make(chan struct{}),
		setErrOnce:       new(sync.Once),
		err:              nil,
		mutex_On:         new(sync.Mutex),
	}
}

func (proto *Protocal) Reply(cmd string, details interface{}) error {
	if proto.isHandshakeDone {
		panic("protoport handshake is done, unexpect Reply call")
	}

	data, err := json.Marshal(&HandshakeOutgoing{
		Command: cmd,
		Details: details,
	})
	if err != nil {
		return err
	}

	_, err = proto.conn.Write(data)
	return err
}

func newReadWriteCloser(buffered io.Reader, conn net.Conn) io.ReadWriteCloser {
	type readWriteCloser struct {
		io.Reader
		io.Writer
		io.Closer
	}

	return &readWriteCloser{
		Reader: io.MultiReader(buffered, conn),
		Writer: conn,
		Closer: conn,
	}
}

var (
	muxadoMutex = new(sync.Mutex)
)

func (proto *Protocal) Multiplex(isClient bool) muxado.Session {
	proto.isHandshakeDone = true

	muxadoMutex.Lock()
	defer muxadoMutex.Unlock()

	if isClient {
		return muxado.Client(newReadWriteCloser(proto.handshakeDecoder.Buffered(), proto.conn), nil)
	}

	return muxado.Server(newReadWriteCloser(proto.handshakeDecoder.Buffered(), proto.conn), nil)
}

func (proto *Protocal) Forward(conn net.Conn) {
	defer proto.conn.Close()
	defer conn.Close()

	proto.isHandshakeDone = true

	wg := &sync.WaitGroup{}
	wg.Add(2)

	go func() {
		defer wg.Done()
		defer conn.Close()
		io.Copy(conn, io.MultiReader(proto.handshakeDecoder.Buffered(), proto.conn))
	}()

	go func() {
		defer wg.Done()
		defer proto.conn.Close()
		io.Copy(proto.conn, conn)
	}()
	wg.Wait()
}

func (proto *Protocal) Request(cmd string, details interface{}) {
	err := proto.Reply(cmd, details)
	if err != nil {
		proto.conn.Close()
		return
	}

	proto.Handle()
}

func (proto *Protocal) Emit(event string, details interface{}) (err error) {
	var data []byte
	data, err = json.Marshal(&HandshakeOutgoing{
		Command: event,
		Details: details,
	})
	if err != nil {
		return err
	}

	var handshake HandshakeIncoming
	err = json.Unmarshal(data, &handshake)
	if err != nil {
		return err
	}

	defer func() {
		if r := recover(); r != nil {
			err = errors.New("conn closed")
		}
	}()
	proto.eventbus <- handshake
	return nil
}

func (proto *Protocal) Handle() {
	defer close(proto.eventbus)
	defer proto.conn.Close()

	if proto.On == nil {
		panic("not set Protocal.On")
	}

	go func() {
		defer proto.conn.Close()

		for handshake := range proto.eventbus {
			err := func() error {
				proto.mutex_On.Lock()
				defer proto.mutex_On.Unlock()
				return proto.On(proto, handshake.Command, handshake.Details)
			}()
			if err != nil {
				proto.shutdown(err)
				return
			}
		}
	}()

	var handshake HandshakeIncoming
	for !proto.isHandshakeDone {
		err := proto.handshakeDecoder.Decode(&handshake)

		if err != nil {
			proto.shutdown(err)
			return
		}
		err = func() error {
			proto.mutex_On.Lock()
			defer proto.mutex_On.Unlock()
			return proto.On(proto, handshake.Command, handshake.Details)
		}()
		if err != nil {
			proto.shutdown(err)
			return
		}

	}

}

func (proto *Protocal) shutdown(err error) {
	if err == nil {
		panic("err cannot be nil")
	}
	proto.setErrOnce.Do(func() {
		proto.err = err
		close(proto.done)
	})
}

func (proto *Protocal) Wait() error {
	<-proto.done
	return proto.err
}
<commit_msg>expose Protocal.shutdown() as .Shutdown()<commit_after>package exposer

import (
	"encoding/json"
	"errors"
	"io"
	"net"
	"sync"

	"github.com/inconshreveable/muxado"
)

type HandshakeHandleFunc func(proto *Protocal, cmd string, details []byte) error
type Protocal struct {
	conn             net.Conn
	isHandshakeDone  bool
	handshakeDecoder *json.Decoder
	eventbus         chan HandshakeIncoming
	done             chan struct{}

	setErrOnce *sync.Once
	err        error

	// handle handshake
	mutex_On *sync.Mutex
	On       HandshakeHandleFunc
}

func NewProtocal(conn net.Conn) *Protocal {
	return &Protocal{
		conn:             conn,
		isHandshakeDone:  false,
		handshakeDecoder: json.NewDecoder(conn),
		eventbus:         make(chan HandshakeIncoming),
		done:             make(chan struct{}),
		setErrOnce:       new(sync.Once),
		err:              nil,
		mutex_On:         new(sync.Mutex),
	}
}

func (proto *Protocal) Reply(cmd string, details interface{}) error {
	if proto.isHandshakeDone {
		panic("protoport handshake is done, unexpect Reply call")
	}

	data, err := json.Marshal(&HandshakeOutgoing{
		Command: cmd,
		Details: details,
	})
	if err != nil {
		return err
	}

	_, err = proto.conn.Write(data)
	return err
}

func newReadWriteCloser(buffered io.Reader, conn net.Conn) io.ReadWriteCloser {
	type readWriteCloser struct {
		io.Reader
		io.Writer
		io.Closer
	}

	return &readWriteCloser{
		Reader: io.MultiReader(buffered, conn),
		Writer: conn,
		Closer: conn,
	}
}

var (
	muxadoMutex = new(sync.Mutex)
)

func (proto *Protocal) Multiplex(isClient bool) muxado.Session {
	proto.isHandshakeDone = true

	muxadoMutex.Lock()
	defer muxadoMutex.Unlock()

	if isClient {
		return muxado.Client(newReadWriteCloser(proto.handshakeDecoder.Buffered(), proto.conn), nil)
	}

	return muxado.Server(newReadWriteCloser(proto.handshakeDecoder.Buffered(), proto.conn), nil)
}

func (proto *Protocal) Forward(conn net.Conn) {
	defer proto.conn.Close()
	defer conn.Close()

	proto.isHandshakeDone = true

	wg := &sync.WaitGroup{}
	wg.Add(2)

	go func() {
		defer wg.Done()
		defer conn.Close()
		io.Copy(conn, io.MultiReader(proto.handshakeDecoder.Buffered(), proto.conn))
	}()

	go func() {
		defer wg.Done()
		defer proto.conn.Close()
		io.Copy(proto.conn, conn)
	}()
	wg.Wait()
}

func (proto *Protocal) Request(cmd string, details interface{}) {
	err := proto.Reply(cmd, details)
	if err != nil {
		proto.conn.Close()
		return
	}

	proto.Handle()
}

func (proto *Protocal) Emit(event string, details interface{}) (err error) {
	var data []byte
	data, err = json.Marshal(&HandshakeOutgoing{
		Command: event,
		Details: details,
	})
	if err != nil {
		return err
	}

	var handshake HandshakeIncoming
	err = json.Unmarshal(data, &handshake)
	if err != nil {
		return err
	}

	defer func() {
		if r := recover(); r != nil {
			err = errors.New("conn closed")
		}
	}()
	proto.eventbus <- handshake
	return nil
}

func (proto *Protocal) Handle() {
	defer close(proto.eventbus)
	defer proto.conn.Close()

	if proto.On == nil {
		panic("not set Protocal.On")
	}

	go func() {
		defer proto.conn.Close()

		for handshake := range proto.eventbus {
			err := func() error {
				proto.mutex_On.Lock()
				defer proto.mutex_On.Unlock()
				return proto.On(proto, handshake.Command, handshake.Details)
			}()
			if err != nil {
				proto.Shutdown(err)
				return
			}
		}
	}()

	var handshake HandshakeIncoming
	for !proto.isHandshakeDone {
		err := proto.handshakeDecoder.Decode(&handshake)

		if err != nil {
			proto.Shutdown(err)
			return
		}
		err = func() error {
			proto.mutex_On.Lock()
			defer proto.mutex_On.Unlock()
			return proto.On(proto, handshake.Command, handshake.Details)
		}()
		if err != nil {
			proto.Shutdown(err)
			return
		}

	}

}

func (proto *Protocal) Shutdown(err error) {
	if err == nil {
		panic("err cannot be nil")
	}
	proto.setErrOnce.Do(func() {
		proto.err = err
		close(proto.done)
	})
}

func (proto *Protocal) Wait() error {
	<-proto.done
	return proto.err
}
<|endoftext|>


<commit_before>/*
	Android Compile Server Frontend.
*/

package main

import (
	"context"
	"flag"
	"html/template"
	"net/http"
	"path/filepath"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/gorilla/mux"

	"go.skia.org/infra/android_compile/go/util"
	"go.skia.org/infra/go/allowed"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/cleanup"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/login"
	"go.skia.org/infra/go/skiaversion"
	"go.skia.org/infra/go/sklog"
)

const (
	FORCE_SYNC_POST_URL = "/_/force_sync"
)

var (
	// Flags
	local        = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	host         = flag.String("host", "localhost", "HTTP service host")
	port         = flag.String("port", ":8000", "HTTP service port.")
	promPort     = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	resourcesDir = flag.String("resources_dir", "", "The directory to find compile.sh and template files.  If blank then the directory two directories up from this source file will be used.")

	// Datastore params
	namespace   = flag.String("namespace", "android-compile-staging", "The Cloud Datastore namespace, such as 'android-compile'.")
	projectName = flag.String("project_name", "google.com:skia-corp", "The Google Cloud project name.")

	// indexTemplate is the main index.html page we serve.
	indexTemplate *template.Template = nil

	serverURL string
)

func reloadTemplates() {
	indexTemplate = template.Must(template.ParseFiles(
		filepath.Join(*resourcesDir, "templates/index.html"),
		filepath.Join(*resourcesDir, "templates/header.html"),
	))
}

func loginHandler(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, login.LoginURL(w, r), http.StatusFound)
	return
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	if *local {
		reloadTemplates()
	}
	w.Header().Set("Content-Type", "text/html")

	if login.LoggedInAs(r) == "" {
		http.Redirect(w, r, login.LoginURL(w, r), http.StatusSeeOther)
		return
	}

	unownedPendingTasks, ownedPendingTasks, err := util.GetPendingCompileTasks("" /* ownedByInstance */)
	if err != nil {
		httputils.ReportError(w, err, "Failed to get unowned/owned compile tasks", http.StatusInternalServerError)
		return
	}
	androidCompileInstances, err := util.GetAllCompileInstances(context.Background())
	if err != nil {
		httputils.ReportError(w, err, "Failed to get android compile instances", http.StatusInternalServerError)
		return
	}

	var info = struct {
		AndroidCompileInstances []*util.AndroidCompileInstance
		UnownedPendingTasks     []*util.CompileTask
		OwnedPendingTasks       []*util.CompileTask
	}{
		AndroidCompileInstances: androidCompileInstances,
		UnownedPendingTasks:     unownedPendingTasks,
		OwnedPendingTasks:       ownedPendingTasks,
	}

	if err := indexTemplate.Execute(w, info); err != nil {
		httputils.ReportError(w, err, "Failed to expand template", http.StatusInternalServerError)
		return
	}
	return
}

func forceSyncHandler(w http.ResponseWriter, r *http.Request) {
	if *local {
		reloadTemplates()
	}

	if login.LoggedInAs(r) == "" {
		http.Redirect(w, r, login.LoginURL(w, r), http.StatusSeeOther)
		return
	}

	if err := util.SetForceMirrorUpdateOnAllInstances(context.Background()); err != nil {
		httputils.ReportError(w, err, "Failed to set force mirror update on all instances", http.StatusInternalServerError)
		return
	}

	sklog.Infof("Force sync button has been pressed by %s", login.LoggedInAs(r))
	return
}

func runServer() {
	r := mux.NewRouter()
	r.PathPrefix("/res/").HandlerFunc(httputils.MakeResourceHandler(*resourcesDir))
	r.HandleFunc("/", indexHandler)
	r.HandleFunc(FORCE_SYNC_POST_URL, forceSyncHandler)

	r.HandleFunc("/json/version", skiaversion.JsonHandler)
	r.HandleFunc(login.DEFAULT_OAUTH2_CALLBACK, login.OAuth2CallbackHandler)
	r.HandleFunc("/login/", loginHandler)
	r.HandleFunc("/logout/", login.LogoutHandler)
	r.HandleFunc("/loginstatus/", login.StatusHandler)
	h := httputils.LoggingGzipRequestResponse(r)
	if !*local {
		h = httputils.HealthzAndHTTPS(h)
	}

	http.Handle("/", h)
	sklog.Infof("Ready to serve on %s", serverURL)
	sklog.Fatal(http.ListenAndServe(*port, nil))
}

func main() {
	flag.Parse()

	common.InitWithMust("android_compile_fe", common.PrometheusOpt(promPort), common.MetricsLoggingOpt())
	defer common.Defer()
	skiaversion.MustLogVersion()

	reloadTemplates()
	serverURL = "https://" + *host
	if *local {
		serverURL = "http://" + *host + *port
	}
	login.InitWithAllow(serverURL+login.DEFAULT_OAUTH2_CALLBACK, allowed.Googlers(), allowed.Googlers(), nil)

	// Create token source.
	ts, err := auth.NewDefaultTokenSource(*local, auth.SCOPE_READ_WRITE, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT, datastore.ScopeDatastore)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}

	// Initialize cloud datastore.
	if err := util.DatastoreInit(*projectName, *namespace, ts); err != nil {
		sklog.Fatalf("Failed to init cloud datastore: %s", err)
	}

	// Start updater for the queue length metrics.
	cleanup.Repeat(time.Minute, func(ctx context.Context) {
		unownedPendingTasks, ownedPendingTasks, err := util.GetPendingCompileTasks("" /* ownedByInstance */)
		if err != nil {
			sklog.Errorf("Failed to get unowned/owned compile tasks: %s", err)
		} else {
			util.QueueLengthMetric.Update(int64(len(unownedPendingTasks)))
			util.RunningLengthMetric.Update(int64(len(ownedPendingTasks)))
		}
	}, nil)

	runServer()
}
<commit_msg>[Android compile] Backend changes necessary for lit-html migration<commit_after>/*
	Android Compile Server Frontend.
*/

package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"net/http"
	"path/filepath"
	"time"

	"cloud.google.com/go/datastore"
	"github.com/gorilla/mux"
	"github.com/unrolled/secure"

	"go.skia.org/infra/android_compile/go/util"
	"go.skia.org/infra/go/allowed"
	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/baseapp"
	"go.skia.org/infra/go/cleanup"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/login"
	"go.skia.org/infra/go/sklog"
)

const (
	forceSyncPostUrl        = "/_/force_sync"
	pendingTasksPostUrl     = "/_/pending_tasks"
	compileInstancesPostUrl = "/_/compile_instances"
)

var (
	// Flags
	host = flag.String("host", "skia-android-compile.corp.goog", "HTTP service host")

	// Datastore params
	namespace   = flag.String("namespace", "android-compile-staging", "The Cloud Datastore namespace, such as 'android-compile'.")
	projectName = flag.String("project_name", "google.com:skia-corp", "The Google Cloud project name.")

	serverURL string
)

// Server is the state of the server.
type Server struct {
	templates *template.Template
}

func (srv *Server) loadTemplates() {
	srv.templates = template.Must(template.New("").Delims("{%", "%}").ParseFiles(
		filepath.Join(*baseapp.ResourcesDir, "index.html"),
	))
}

func (srv *Server) indexHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")

	if err := srv.templates.ExecuteTemplate(w, "index.html", map[string]string{
		// Look in webpack.config.js for where the nonce templates are injected.
		"Nonce": secure.CSPNonce(r.Context()),
	}); err != nil {
		httputils.ReportError(w, err, "Failed to expand template", http.StatusInternalServerError)
		return
	}
}

func (srv *Server) compileInstancesHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	compileInstances, err := util.GetAllCompileInstances(context.Background())
	if err != nil {
		httputils.ReportError(w, err, "Failed to get android compile instances", http.StatusInternalServerError)
		return
	}

	if err := json.NewEncoder(w).Encode(compileInstances); err != nil {
		httputils.ReportError(w, err, fmt.Sprintf("Failed to encode JSON: %v", err), http.StatusInternalServerError)
		return
	}
}

func (srv *Server) pendingTasksHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	unownedPendingTasks, ownedPendingTasks, err := util.GetPendingCompileTasks("" /* ownedByInstance */)
	if err != nil {
		httputils.ReportError(w, err, "Failed to get unowned/owned compile tasks", http.StatusInternalServerError)
		return
	}

	var pendingTasks = struct {
		UnOwnedPendingTasks []*util.CompileTask `json:"unowned_pending_tasks"`
		OwnedPendingTasks   []*util.CompileTask `json:"owned_pending_tasks"`
	}{
		UnOwnedPendingTasks: unownedPendingTasks,
		OwnedPendingTasks:   ownedPendingTasks,
	}

	if err := json.NewEncoder(w).Encode(pendingTasks); err != nil {
		httputils.ReportError(w, err, fmt.Sprintf("Failed to encode JSON: %v", err), http.StatusInternalServerError)
		return
	}
}

func (srv *Server) forceSyncHandler(w http.ResponseWriter, r *http.Request) {
	if err := util.SetForceMirrorUpdateOnAllInstances(context.Background()); err != nil {
		httputils.ReportError(w, err, "Failed to set force mirror update on all instances", http.StatusInternalServerError)
		return
	}
	sklog.Infof("Force sync button has been pressed by %s", login.LoggedInAs(r))

	compileInstances, err := util.GetAllCompileInstances(context.Background())
	if err != nil {
		httputils.ReportError(w, err, "Failed to get android compile instances", http.StatusInternalServerError)
		return
	}

	if err := json.NewEncoder(w).Encode(compileInstances); err != nil {
		httputils.ReportError(w, err, fmt.Sprintf("Failed to encode JSON: %v", err), http.StatusInternalServerError)
		return
	}
	return
}

// See baseapp.App.
func (srv *Server) AddHandlers(r *mux.Router) {
	// For login/logout.
	r.HandleFunc(login.DEFAULT_OAUTH2_CALLBACK, login.OAuth2CallbackHandler)
	r.HandleFunc("/logout/", login.LogoutHandler)
	r.HandleFunc("/loginstatus/", login.StatusHandler)
	// The main page.
	r.HandleFunc("/", srv.indexHandler).Methods("GET")
	// POST handlers.
	r.HandleFunc(forceSyncPostUrl, srv.forceSyncHandler).Methods("POST")
	r.HandleFunc(pendingTasksPostUrl, srv.pendingTasksHandler).Methods("POST")
	r.HandleFunc(compileInstancesPostUrl, srv.compileInstancesHandler).Methods("POST")
}

// See baseapp.Constructor
func New() (baseapp.App, error) {
	login.SimpleInitWithAllow(*baseapp.Port, *baseapp.Local, allowed.Googlers(), allowed.Googlers(), nil)

	// Create token source.
	ts, err := auth.NewDefaultTokenSource(*baseapp.Local, auth.SCOPE_READ_WRITE, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT, datastore.ScopeDatastore)
	if err != nil {
		sklog.Fatalf("Problem setting up default token source: %s", err)
	}

	// Initialize cloud datastore.
	if err := util.DatastoreInit(*projectName, *namespace, ts); err != nil {
		sklog.Fatalf("Failed to init cloud datastore: %s", err)
	}

	// Start updater for the queue length metrics.
	cleanup.Repeat(time.Minute, func(ctx context.Context) {
		unownedPendingTasks, ownedPendingTasks, err := util.GetPendingCompileTasks("" /* ownedByInstance */)
		if err != nil {
			sklog.Errorf("Failed to get unowned/owned compile tasks: %s", err)
		} else {
			util.QueueLengthMetric.Update(int64(len(unownedPendingTasks)))
			util.RunningLengthMetric.Update(int64(len(ownedPendingTasks)))
		}
	}, nil)

	srv := &Server{}
	srv.loadTemplates()

	return srv, nil
}

// See baseapp.App.
func (srv *Server) AddMiddleware() []mux.MiddlewareFunc {
	ret := []mux.MiddlewareFunc{}
	if !*baseapp.Local {
		ret = append(ret, login.ForceAuthMiddleware(login.DEFAULT_REDIRECT_URL), login.RestrictViewer)
	}
	return ret
}

func main() {
	baseapp.Serve(New, []string{*host})
}
<|endoftext|>