<commit_before>package main

import (
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
)

const (
	serverSoftware   = "websocketd/0.0.0.0" // TODO: Update this at each release.
	gatewayInterface = "websocketd-CGI/0.1"
)

var headerNewlineToSpace = strings.NewReplacer("\n", " ", "\r", " ")
var headerDashToUnderscore = strings.NewReplacer("-", "_")

func createEnv(ws *websocket.Conn, config *Config) ([]string, error) {
	req := ws.Request()
	headers := req.Header
	url := req.URL

	remoteAddr, remotePort, err := net.SplitHostPort(req.RemoteAddr)
	if err != nil {
		return nil, err
	}

	var remoteHost string
	if config.ReverseLookup {
		remoteHosts, err := net.LookupAddr(remoteAddr)
		if err != nil {
			return nil, err
		}
		if len(remoteHosts) == 0 {
			remoteHost = remoteAddr
		} else {
			remoteHost = remoteHosts[0]
		}
	} else {
		remoteHost = remoteAddr
	}

	serverName, serverPort, err := net.SplitHostPort(req.Host)
	if err != nil {
		return nil, err
	}

	uniqueId := time.Now().UnixNano() // Just use this a unique counter.

	standardEnvCount := 14
	env := make([]string, 0, len(headers)+standardEnvCount)

	// IMPORTANT ---> Adding a header? Make sure standardHeaderCount (above) is up to date.

	// Standard CGI specification headers.
	// As defined in http://tools.ietf.org/html/rfc3875
	env = appendEnv(env, "REMOTE_ADDR", remoteAddr)
	env = appendEnv(env, "REMOTE_HOST", remoteHost)
	env = appendEnv(env, "SERVER_NAME", serverName)
	env = appendEnv(env, "SERVER_PORT", serverPort)
	env = appendEnv(env, "SERVER_PROTOCOL", req.Proto)
	env = appendEnv(env, "SERVER_SOFTWARE", serverSoftware)
	env = appendEnv(env, "GATEWAY_INTERFACE", gatewayInterface)
	env = appendEnv(env, "REQUEST_METHOD", req.Method)
	env = appendEnv(env, "SCRIPT_NAME", "/")
	env = appendEnv(env, "PATH_INFO", url.Path)
	env = appendEnv(env, "PATH_TRANSLATED", url.Path)
	env = appendEnv(env, "QUERY_STRING", url.RawQuery)

	// Non standard, but commonly used headers.
	env = appendEnv(env, "UNIQUE_ID", strconv.FormatInt(uniqueId, 10)) // Based on Apache mod_unique_id.
	env = appendEnv(env, "REMOTE_PORT", remotePort)
	env = appendEnv(env, "REQUEST_URI", url.RequestURI()) // e.g. /foo/blah?a=b

	// The following variables are part of the CGI specification, but are optional
	// and not set by websocketd:
	//
	//   AUTH_TYPE, REMOTE_USER, REMOTE_IDENT
	//     -- Authentication left to the underlying programs.
	//
	//   CONTENT_LENGTH, CONTENT_TYPE
	//     -- makes no sense for WebSocket connections.
	//
	//   HTTPS, SSL_*
	//     -- SSL not supported

	for k, _ := range headers {
		env = appendEnv(env, fmt.Sprintf("HTTP_%s", headerDashToUnderscore.Replace(k)), headers[k]...)
	}

	return env, nil
}

// Adapted from net/http/header.go
func appendEnv(env []string, k string, v ...string) []string {
	if len(v) == 0 {
		return env
	}
	vCleaned := make([]string, 0, len(v))
	for _, val := range v {
		vCleaned = append(vCleaned, strings.TrimSpace(headerNewlineToSpace.Replace(val)))
	}
	return append(env, fmt.Sprintf("%s=%s",
		strings.ToUpper(k),
		strings.Join(vCleaned, ", ")))
}
<commit_msg>Explicitly set unused CGI variables to empty.<commit_after>package main

import (
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
)

const (
	serverSoftware   = "websocketd/0.0.0.0" // TODO: Update this at each release.
	gatewayInterface = "websocketd-CGI/0.1"
)

var headerNewlineToSpace = strings.NewReplacer("\n", " ", "\r", " ")
var headerDashToUnderscore = strings.NewReplacer("-", "_")

func createEnv(ws *websocket.Conn, config *Config) ([]string, error) {
	req := ws.Request()
	headers := req.Header
	url := req.URL

	remoteAddr, remotePort, err := net.SplitHostPort(req.RemoteAddr)
	if err != nil {
		return nil, err
	}

	var remoteHost string
	if config.ReverseLookup {
		remoteHosts, err := net.LookupAddr(remoteAddr)
		if err != nil {
			return nil, err
		}
		if len(remoteHosts) == 0 {
			remoteHost = remoteAddr
		} else {
			remoteHost = remoteHosts[0]
		}
	} else {
		remoteHost = remoteAddr
	}

	serverName, serverPort, err := net.SplitHostPort(req.Host)
	if err != nil {
		return nil, err
	}

	uniqueId := time.Now().UnixNano() // Just use this a unique counter.

	standardEnvCount := 20
	env := make([]string, 0, len(headers)+standardEnvCount)

	// IMPORTANT ---> Adding a header? Make sure standardHeaderCount (above) is up to date.

	// Standard CGI specification headers.
	// As defined in http://tools.ietf.org/html/rfc3875
	env = appendEnv(env, "REMOTE_ADDR", remoteAddr)
	env = appendEnv(env, "REMOTE_HOST", remoteHost)
	env = appendEnv(env, "SERVER_NAME", serverName)
	env = appendEnv(env, "SERVER_PORT", serverPort)
	env = appendEnv(env, "SERVER_PROTOCOL", req.Proto)
	env = appendEnv(env, "SERVER_SOFTWARE", serverSoftware)
	env = appendEnv(env, "GATEWAY_INTERFACE", gatewayInterface)
	env = appendEnv(env, "REQUEST_METHOD", req.Method)
	env = appendEnv(env, "SCRIPT_NAME", "/")
	env = appendEnv(env, "PATH_INFO", url.Path)
	env = appendEnv(env, "PATH_TRANSLATED", url.Path)
	env = appendEnv(env, "QUERY_STRING", url.RawQuery)

	// Not supported, but we explicitly clear them so we don't get leaks from parent environment.
	env = appendEnv(env, "AUTH_TYPE", "")
	env = appendEnv(env, "CONTENT_LENGTH", "")
	env = appendEnv(env, "CONTENT_TYPE", "")
	env = appendEnv(env, "REMOTE_IDENT", "")
	env = appendEnv(env, "REMOTE_USER", "")

	// Non standard, but commonly used headers.
	env = appendEnv(env, "UNIQUE_ID", strconv.FormatInt(uniqueId, 10)) // Based on Apache mod_unique_id.
	env = appendEnv(env, "REMOTE_PORT", remotePort)
	env = appendEnv(env, "REQUEST_URI", url.RequestURI()) // e.g. /foo/blah?a=b

	// The following variables are part of the CGI specification, but are optional
	// and not set by websocketd:
	//
	//   AUTH_TYPE, REMOTE_USER, REMOTE_IDENT
	//     -- Authentication left to the underlying programs.
	//
	//   CONTENT_LENGTH, CONTENT_TYPE
	//     -- makes no sense for WebSocket connections.
	//
	//   HTTPS, SSL_*
	//     -- SSL not supported

	for k, _ := range headers {
		env = appendEnv(env, fmt.Sprintf("HTTP_%s", headerDashToUnderscore.Replace(k)), headers[k]...)
	}

	return env, nil
}

// Adapted from net/http/header.go
func appendEnv(env []string, k string, v ...string) []string {
	if len(v) == 0 {
		return env
	}
	vCleaned := make([]string, 0, len(v))
	for _, val := range v {
		vCleaned = append(vCleaned, strings.TrimSpace(headerNewlineToSpace.Replace(val)))
	}
	return append(env, fmt.Sprintf("%s=%s",
		strings.ToUpper(k),
		strings.Join(vCleaned, ", ")))
}
<|endoftext|>


<commit_before>package mdb

/*
#include <stdlib.h>
#include <lmdb.h>
*/
import "C"

import (
	"errors"
	"fmt"
	"syscall"
	"unsafe"
)

const SUCCESS = C.MDB_SUCCESS

// mdb_env Environment Flags
const (
	FIXEDMAP   = C.MDB_FIXEDMAP   // mmap at a fixed address (experimental)
	NOSUBDIR   = C.MDB_NOSUBDIR   // no environment directory
	NOSYNC     = C.MDB_NOSYNC     // don't fsync after commit
	RDONLY     = C.MDB_RDONLY     // read only
	NOMETASYNC = C.MDB_NOMETASYNC // don't fsync metapage after commit
	WRITEMAP   = C.MDB_WRITEMAP   // use writable mmap
	MAPASYNC   = C.MDB_MAPASYNC   // use asynchronous msync when MDB_WRITEMAP is use
	NOTLS      = C.MDB_NOTLS      // tie reader locktable slots to Txn objects instead of threads
	NORDAHEAD  = C.MDB_NORDAHEAD  // turn off readahead
)

type DBI uint

type Errno C.int

// minimum and maximum values produced for the Errno type. syscall.Errnos of
// other values may still be produced.
const minErrno, maxErrno C.int = C.MDB_KEYEXIST, C.MDB_LAST_ERRCODE

func (e Errno) Error() string {
	s := C.GoString(C.mdb_strerror(C.int(e)))
	if s == "" {
		return fmt.Sprint("mdb errno:", int(e))
	}
	return s
}

// for tests that can't import C
func _errno(ret int) error {
	return errno(C.int(ret))
}

func errno(ret C.int) error {
	if ret == C.MDB_SUCCESS {
		return nil
	}
	if minErrno <= ret && ret <= maxErrno {
		return Errno(ret)
	}
	return syscall.Errno(ret)
}

// error codes
const (
	KeyExist        = Errno(C.MDB_KEYEXIST)
	NotFound        = Errno(C.MDB_NOTFOUND)
	PageNotFound    = Errno(C.MDB_PAGE_NOTFOUND)
	Corrupted       = Errno(C.MDB_CORRUPTED)
	Panic           = Errno(C.MDB_PANIC)
	VersionMismatch = Errno(C.MDB_VERSION_MISMATCH)
	Invalid         = Errno(C.MDB_INVALID)
	MapFull         = Errno(C.MDB_MAP_FULL)
	DbsFull         = Errno(C.MDB_DBS_FULL)
	ReadersFull     = Errno(C.MDB_READERS_FULL)
	TlsFull         = Errno(C.MDB_TLS_FULL)
	TxnFull         = Errno(C.MDB_TXN_FULL)
	CursorFull      = Errno(C.MDB_CURSOR_FULL)
	PageFull        = Errno(C.MDB_PAGE_FULL)
	MapResized      = Errno(C.MDB_MAP_RESIZED)
	Incompatibile   = Errno(C.MDB_INCOMPATIBLE)
)

func Version() string {
	var major, minor, patch *C.int
	ver_str := C.mdb_version(major, minor, patch)
	return C.GoString(ver_str)
}

// Env is opaque structure for a database environment.
// A DB environment supports multiple databases, all residing in the
// same shared-memory map.
type Env struct {
	env *C.MDB_env
}

// Create an MDB environment handle.
func NewEnv() (*Env, error) {
	var env *C.MDB_env
	ret := C.mdb_env_create(&env)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	return &Env{env}, nil
}

// Open an environment handle. If this function fails Close() must be called to discard the Env handle.
func (env *Env) Open(path string, flags uint, mode uint) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_open(env.env, cpath, C.uint(NOTLS|flags), C.mdb_mode_t(mode))
	return errno(ret)
}

func (env *Env) Close() error {
	if env.env == nil {
		return errors.New("Environment already closed")
	}
	C.mdb_env_close(env.env)
	env.env = nil
	return nil
}

func (env *Env) Copy(path string) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_copy(env.env, cpath)
	return errno(ret)
}

// Statistics for a database in the environment
type Stat struct {
	PSize         uint   // Size of a database page. This is currently the same for all databases.
	Depth         uint   // Depth (height) of the B-tree
	BranchPages   uint64 // Number of internal (non-leaf) pages
	LeafPages     uint64 // Number of leaf pages
	OverflowPages uint64 // Number of overflow pages
	Entries       uint64 // Number of data items
}

func (env *Env) Stat() (*Stat, error) {
	var c_stat C.MDB_stat
	ret := C.mdb_env_stat(env.env, &c_stat)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	stat := Stat{PSize: uint(c_stat.ms_psize),
		Depth:         uint(c_stat.ms_depth),
		BranchPages:   uint64(c_stat.ms_branch_pages),
		LeafPages:     uint64(c_stat.ms_leaf_pages),
		OverflowPages: uint64(c_stat.ms_overflow_pages),
		Entries:       uint64(c_stat.ms_entries)}
	return &stat, nil
}

type Info struct {
	MapSize    uint64 // Size of the data memory map
	LastPNO    uint64 // ID of the last used page
	LastTxnID  uint64 // ID of the last committed transaction
	MaxReaders uint   // maximum number of threads for the environment
	NumReaders uint   // maximum number of threads used in the environment
}

func (env *Env) Info() (*Info, error) {
	var c_info C.MDB_envinfo
	ret := C.mdb_env_info(env.env, &c_info)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	info := Info{MapSize: uint64(c_info.me_mapsize),
		LastPNO:    uint64(c_info.me_last_pgno),
		LastTxnID:  uint64(c_info.me_last_txnid),
		MaxReaders: uint(c_info.me_maxreaders),
		NumReaders: uint(c_info.me_numreaders)}
	return &info, nil
}

func (env *Env) Sync(force int) error {
	ret := C.mdb_env_sync(env.env, C.int(force))
	return errno(ret)
}

func (env *Env) SetFlags(flags uint, onoff int) error {
	ret := C.mdb_env_set_flags(env.env, C.uint(flags), C.int(onoff))
	return errno(ret)
}

func (env *Env) Flags() (uint, error) {
	var flags C.uint
	ret := C.mdb_env_get_flags(env.env, &flags)
	if ret != SUCCESS {
		return 0, errno(ret)
	}
	return uint(flags), nil
}

func (env *Env) Path() (string, error) {
	var path string
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_get_path(env.env, &cpath)
	if ret != SUCCESS {
		return "", errno(ret)
	}
	return C.GoString(cpath), nil
}

func (env *Env) SetMapSize(size uint64) error {
	ret := C.mdb_env_set_mapsize(env.env, C.size_t(size))
	return errno(ret)
}

func (env *Env) SetMaxReaders(size uint) error {
	ret := C.mdb_env_set_maxreaders(env.env, C.uint(size))
	return errno(ret)
}

func (env *Env) SetMaxDBs(size DBI) error {
	ret := C.mdb_env_set_maxdbs(env.env, C.MDB_dbi(size))
	return errno(ret)
}

func (env *Env) DBIClose(dbi DBI) {
	C.mdb_dbi_close(env.env, C.MDB_dbi(dbi))
}
<commit_msg>Grrr. Can't believe I've only just spotted this hard coding of NOTLS. Major design decision of this binding is to avoid needing the NOTLS flag.<commit_after>package mdb

/*
#include <stdlib.h>
#include <lmdb.h>
*/
import "C"

import (
	"errors"
	"fmt"
	"syscall"
	"unsafe"
)

const SUCCESS = C.MDB_SUCCESS

// mdb_env Environment Flags
const (
	FIXEDMAP   = C.MDB_FIXEDMAP   // mmap at a fixed address (experimental)
	NOSUBDIR   = C.MDB_NOSUBDIR   // no environment directory
	NOSYNC     = C.MDB_NOSYNC     // don't fsync after commit
	RDONLY     = C.MDB_RDONLY     // read only
	NOMETASYNC = C.MDB_NOMETASYNC // don't fsync metapage after commit
	WRITEMAP   = C.MDB_WRITEMAP   // use writable mmap
	MAPASYNC   = C.MDB_MAPASYNC   // use asynchronous msync when MDB_WRITEMAP is use
	NOTLS      = C.MDB_NOTLS      // tie reader locktable slots to Txn objects instead of threads
	NORDAHEAD  = C.MDB_NORDAHEAD  // turn off readahead
)

type DBI uint

type Errno C.int

// minimum and maximum values produced for the Errno type. syscall.Errnos of
// other values may still be produced.
const minErrno, maxErrno C.int = C.MDB_KEYEXIST, C.MDB_LAST_ERRCODE

func (e Errno) Error() string {
	s := C.GoString(C.mdb_strerror(C.int(e)))
	if s == "" {
		return fmt.Sprint("mdb errno:", int(e))
	}
	return s
}

// for tests that can't import C
func _errno(ret int) error {
	return errno(C.int(ret))
}

func errno(ret C.int) error {
	if ret == C.MDB_SUCCESS {
		return nil
	}
	if minErrno <= ret && ret <= maxErrno {
		return Errno(ret)
	}
	return syscall.Errno(ret)
}

// error codes
const (
	KeyExist        = Errno(C.MDB_KEYEXIST)
	NotFound        = Errno(C.MDB_NOTFOUND)
	PageNotFound    = Errno(C.MDB_PAGE_NOTFOUND)
	Corrupted       = Errno(C.MDB_CORRUPTED)
	Panic           = Errno(C.MDB_PANIC)
	VersionMismatch = Errno(C.MDB_VERSION_MISMATCH)
	Invalid         = Errno(C.MDB_INVALID)
	MapFull         = Errno(C.MDB_MAP_FULL)
	DbsFull         = Errno(C.MDB_DBS_FULL)
	ReadersFull     = Errno(C.MDB_READERS_FULL)
	TlsFull         = Errno(C.MDB_TLS_FULL)
	TxnFull         = Errno(C.MDB_TXN_FULL)
	CursorFull      = Errno(C.MDB_CURSOR_FULL)
	PageFull        = Errno(C.MDB_PAGE_FULL)
	MapResized      = Errno(C.MDB_MAP_RESIZED)
	Incompatibile   = Errno(C.MDB_INCOMPATIBLE)
)

func Version() string {
	var major, minor, patch *C.int
	ver_str := C.mdb_version(major, minor, patch)
	return C.GoString(ver_str)
}

// Env is opaque structure for a database environment.
// A DB environment supports multiple databases, all residing in the
// same shared-memory map.
type Env struct {
	env *C.MDB_env
}

// Create an MDB environment handle.
func NewEnv() (*Env, error) {
	var env *C.MDB_env
	ret := C.mdb_env_create(&env)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	return &Env{env}, nil
}

// Open an environment handle. If this function fails Close() must be called to discard the Env handle.
func (env *Env) Open(path string, flags uint, mode uint) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_open(env.env, cpath, C.uint(flags), C.mdb_mode_t(mode))
	return errno(ret)
}

func (env *Env) Close() error {
	if env.env == nil {
		return errors.New("Environment already closed")
	}
	C.mdb_env_close(env.env)
	env.env = nil
	return nil
}

func (env *Env) Copy(path string) error {
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_copy(env.env, cpath)
	return errno(ret)
}

// Statistics for a database in the environment
type Stat struct {
	PSize         uint   // Size of a database page. This is currently the same for all databases.
	Depth         uint   // Depth (height) of the B-tree
	BranchPages   uint64 // Number of internal (non-leaf) pages
	LeafPages     uint64 // Number of leaf pages
	OverflowPages uint64 // Number of overflow pages
	Entries       uint64 // Number of data items
}

func (env *Env) Stat() (*Stat, error) {
	var c_stat C.MDB_stat
	ret := C.mdb_env_stat(env.env, &c_stat)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	stat := Stat{PSize: uint(c_stat.ms_psize),
		Depth:         uint(c_stat.ms_depth),
		BranchPages:   uint64(c_stat.ms_branch_pages),
		LeafPages:     uint64(c_stat.ms_leaf_pages),
		OverflowPages: uint64(c_stat.ms_overflow_pages),
		Entries:       uint64(c_stat.ms_entries)}
	return &stat, nil
}

type Info struct {
	MapSize    uint64 // Size of the data memory map
	LastPNO    uint64 // ID of the last used page
	LastTxnID  uint64 // ID of the last committed transaction
	MaxReaders uint   // maximum number of threads for the environment
	NumReaders uint   // maximum number of threads used in the environment
}

func (env *Env) Info() (*Info, error) {
	var c_info C.MDB_envinfo
	ret := C.mdb_env_info(env.env, &c_info)
	if ret != SUCCESS {
		return nil, errno(ret)
	}
	info := Info{MapSize: uint64(c_info.me_mapsize),
		LastPNO:    uint64(c_info.me_last_pgno),
		LastTxnID:  uint64(c_info.me_last_txnid),
		MaxReaders: uint(c_info.me_maxreaders),
		NumReaders: uint(c_info.me_numreaders)}
	return &info, nil
}

func (env *Env) Sync(force int) error {
	ret := C.mdb_env_sync(env.env, C.int(force))
	return errno(ret)
}

func (env *Env) SetFlags(flags uint, onoff int) error {
	ret := C.mdb_env_set_flags(env.env, C.uint(flags), C.int(onoff))
	return errno(ret)
}

func (env *Env) Flags() (uint, error) {
	var flags C.uint
	ret := C.mdb_env_get_flags(env.env, &flags)
	if ret != SUCCESS {
		return 0, errno(ret)
	}
	return uint(flags), nil
}

func (env *Env) Path() (string, error) {
	var path string
	cpath := C.CString(path)
	defer C.free(unsafe.Pointer(cpath))
	ret := C.mdb_env_get_path(env.env, &cpath)
	if ret != SUCCESS {
		return "", errno(ret)
	}
	return C.GoString(cpath), nil
}

func (env *Env) SetMapSize(size uint64) error {
	ret := C.mdb_env_set_mapsize(env.env, C.size_t(size))
	return errno(ret)
}

func (env *Env) SetMaxReaders(size uint) error {
	ret := C.mdb_env_set_maxreaders(env.env, C.uint(size))
	return errno(ret)
}

func (env *Env) SetMaxDBs(size DBI) error {
	ret := C.mdb_env_set_maxdbs(env.env, C.MDB_dbi(size))
	return errno(ret)
}

func (env *Env) DBIClose(dbi DBI) {
	C.mdb_dbi_close(env.env, C.MDB_dbi(dbi))
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// failIgnore is a set of builders that we don't email about because
// they're too flaky.
var failIgnore = map[string]bool{
	"netbsd-386-bsiegert":   true,
	"netbsd-amd64-bsiegert": true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if failIgnore[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	var err error
	if broken != nil && !broken.FailNotificationSent {
		c.Infof("%s is broken commit; notifying", broken.Hash)
		notifyLater.Call(c, broken, builder) // add task to queue
		broken.FailNotificationSent = true
		_, err = datastore.Put(c, broken.Key(c), broken)
	}
	return err
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder string) {
	if !updateCL(c, com, builder) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder)
	}
}

// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder string) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	url := fmt.Sprintf("%v?cl=%v&brokebuild=%v", gobotBase, cl, builder)
	r, err := urlfetch.Client(c).Post(url, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder string) {
	// TODO(adg): handle packages

	// get Result
	r := com.Result(builder, "")
	if r == nil {
		c.Errorf("finding result for %q: %+v", builder, com)
		return
	}

	// get Log
	k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", r.LogHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Result": r, "Log": l,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<commit_msg>go.tools/dashboard/app: update bad builder list<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// ignoreFailure is a set of builders that we don't email about because
// they are not yet production-ready.
var ignoreFailure = map[string]bool{
	"dragonfly-amd64":       true,
	"netbsd-amd64-bsiegert": true,
	"plan-386-cnielsen":     true,
	"solaris-amd64-smartos": true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if ignoreFailure[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	var err error
	if broken != nil && !broken.FailNotificationSent {
		c.Infof("%s is broken commit; notifying", broken.Hash)
		notifyLater.Call(c, broken, builder) // add task to queue
		broken.FailNotificationSent = true
		_, err = datastore.Put(c, broken.Key(c), broken)
	}
	return err
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder string) {
	if !updateCL(c, com, builder) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder)
	}
}

// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder string) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	url := fmt.Sprintf("%v?cl=%v&brokebuild=%v", gobotBase, cl, builder)
	r, err := urlfetch.Client(c).Post(url, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder string) {
	// TODO(adg): handle packages

	// get Result
	r := com.Result(builder, "")
	if r == nil {
		c.Errorf("finding result for %q: %+v", builder, com)
		return
	}

	// get Log
	k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", r.LogHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Result": r, "Log": l,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<|endoftext|>


<commit_before>package all

import (
	_ "github.com/influxdata/telegraf/plugins/inputs/aerospike"
	_ "github.com/influxdata/telegraf/plugins/inputs/apache"
	_ "github.com/influxdata/telegraf/plugins/inputs/bcache"
	_ "github.com/influxdata/telegraf/plugins/inputs/cassandra"
	_ "github.com/influxdata/telegraf/plugins/inputs/ceph"
	_ "github.com/influxdata/telegraf/plugins/inputs/cgroup"
	_ "github.com/influxdata/telegraf/plugins/inputs/chrony"
	_ "github.com/influxdata/telegraf/plugins/inputs/cloudwatch"
	_ "github.com/influxdata/telegraf/plugins/inputs/conntrack"
	_ "github.com/influxdata/telegraf/plugins/inputs/consul"
	_ "github.com/influxdata/telegraf/plugins/inputs/couchbase"
	_ "github.com/influxdata/telegraf/plugins/inputs/couchdb"
	_ "github.com/influxdata/telegraf/plugins/inputs/disque"
	_ "github.com/influxdata/telegraf/plugins/inputs/dns_query"
	_ "github.com/influxdata/telegraf/plugins/inputs/docker"
	_ "github.com/influxdata/telegraf/plugins/inputs/dovecot"
	_ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch"
	_ "github.com/influxdata/telegraf/plugins/inputs/exec"
	_ "github.com/influxdata/telegraf/plugins/inputs/filestat"
	_ "github.com/influxdata/telegraf/plugins/inputs/graylog"
	_ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
	_ "github.com/influxdata/telegraf/plugins/inputs/http_listener"
	_ "github.com/influxdata/telegraf/plugins/inputs/hddtemp"
	_ "github.com/influxdata/telegraf/plugins/inputs/http_response"
	_ "github.com/influxdata/telegraf/plugins/inputs/httpjson"
	_ "github.com/influxdata/telegraf/plugins/inputs/influxdb"
	_ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor"
	_ "github.com/influxdata/telegraf/plugins/inputs/jolokia"
	_ "github.com/influxdata/telegraf/plugins/inputs/kafka_consumer"
	_ "github.com/influxdata/telegraf/plugins/inputs/leofs"
	_ "github.com/influxdata/telegraf/plugins/inputs/logparser"
	_ "github.com/influxdata/telegraf/plugins/inputs/lustre2"
	_ "github.com/influxdata/telegraf/plugins/inputs/mailchimp"
	_ "github.com/influxdata/telegraf/plugins/inputs/memcached"
	_ "github.com/influxdata/telegraf/plugins/inputs/mesos"
	_ "github.com/influxdata/telegraf/plugins/inputs/mongodb"
	_ "github.com/influxdata/telegraf/plugins/inputs/mqtt_consumer"
	_ "github.com/influxdata/telegraf/plugins/inputs/mysql"
	_ "github.com/influxdata/telegraf/plugins/inputs/nats_consumer"
	_ "github.com/influxdata/telegraf/plugins/inputs/net_response"
	_ "github.com/influxdata/telegraf/plugins/inputs/nginx"
	_ "github.com/influxdata/telegraf/plugins/inputs/nsq"
	_ "github.com/influxdata/telegraf/plugins/inputs/nsq_consumer"
	_ "github.com/influxdata/telegraf/plugins/inputs/nstat"
	_ "github.com/influxdata/telegraf/plugins/inputs/ntpq"
	_ "github.com/influxdata/telegraf/plugins/inputs/passenger"
	_ "github.com/influxdata/telegraf/plugins/inputs/phpfpm"
	_ "github.com/influxdata/telegraf/plugins/inputs/ping"
	_ "github.com/influxdata/telegraf/plugins/inputs/postgresql"
	_ "github.com/influxdata/telegraf/plugins/inputs/postgresql_extensible"
	_ "github.com/influxdata/telegraf/plugins/inputs/powerdns"
	_ "github.com/influxdata/telegraf/plugins/inputs/procstat"
	_ "github.com/influxdata/telegraf/plugins/inputs/prometheus"
	_ "github.com/influxdata/telegraf/plugins/inputs/puppetagent"
	_ "github.com/influxdata/telegraf/plugins/inputs/rabbitmq"
	_ "github.com/influxdata/telegraf/plugins/inputs/raindrops"
	_ "github.com/influxdata/telegraf/plugins/inputs/redis"
	_ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb"
	_ "github.com/influxdata/telegraf/plugins/inputs/riak"
	_ "github.com/influxdata/telegraf/plugins/inputs/sensors"
	_ "github.com/influxdata/telegraf/plugins/inputs/snmp"
	_ "github.com/influxdata/telegraf/plugins/inputs/sqlserver"
	_ "github.com/influxdata/telegraf/plugins/inputs/statsd"
	_ "github.com/influxdata/telegraf/plugins/inputs/sysstat"
	_ "github.com/influxdata/telegraf/plugins/inputs/system"
	_ "github.com/influxdata/telegraf/plugins/inputs/tail"
	_ "github.com/influxdata/telegraf/plugins/inputs/tcp_listener"
	_ "github.com/influxdata/telegraf/plugins/inputs/trig"
	_ "github.com/influxdata/telegraf/plugins/inputs/twemproxy"
	_ "github.com/influxdata/telegraf/plugins/inputs/udp_listener"
	_ "github.com/influxdata/telegraf/plugins/inputs/varnish"
	_ "github.com/influxdata/telegraf/plugins/inputs/webhooks"
	_ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters"
	_ "github.com/influxdata/telegraf/plugins/inputs/zfs"
	_ "github.com/influxdata/telegraf/plugins/inputs/zookeeper"
)
<commit_msg>correct linter error after merge<commit_after>package all

import (
	_ "github.com/influxdata/telegraf/plugins/inputs/aerospike"
	_ "github.com/influxdata/telegraf/plugins/inputs/apache"
	_ "github.com/influxdata/telegraf/plugins/inputs/bcache"
	_ "github.com/influxdata/telegraf/plugins/inputs/cassandra"
	_ "github.com/influxdata/telegraf/plugins/inputs/ceph"
	_ "github.com/influxdata/telegraf/plugins/inputs/cgroup"
	_ "github.com/influxdata/telegraf/plugins/inputs/chrony"
	_ "github.com/influxdata/telegraf/plugins/inputs/cloudwatch"
	_ "github.com/influxdata/telegraf/plugins/inputs/conntrack"
	_ "github.com/influxdata/telegraf/plugins/inputs/consul"
	_ "github.com/influxdata/telegraf/plugins/inputs/couchbase"
	_ "github.com/influxdata/telegraf/plugins/inputs/couchdb"
	_ "github.com/influxdata/telegraf/plugins/inputs/disque"
	_ "github.com/influxdata/telegraf/plugins/inputs/dns_query"
	_ "github.com/influxdata/telegraf/plugins/inputs/docker"
	_ "github.com/influxdata/telegraf/plugins/inputs/dovecot"
	_ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch"
	_ "github.com/influxdata/telegraf/plugins/inputs/exec"
	_ "github.com/influxdata/telegraf/plugins/inputs/filestat"
	_ "github.com/influxdata/telegraf/plugins/inputs/graylog"
	_ "github.com/influxdata/telegraf/plugins/inputs/haproxy"
	_ "github.com/influxdata/telegraf/plugins/inputs/hddtemp"
	_ "github.com/influxdata/telegraf/plugins/inputs/http_listener"
	_ "github.com/influxdata/telegraf/plugins/inputs/http_response"
	_ "github.com/influxdata/telegraf/plugins/inputs/httpjson"
	_ "github.com/influxdata/telegraf/plugins/inputs/influxdb"
	_ "github.com/influxdata/telegraf/plugins/inputs/ipmi_sensor"
	_ "github.com/influxdata/telegraf/plugins/inputs/jolokia"
	_ "github.com/influxdata/telegraf/plugins/inputs/kafka_consumer"
	_ "github.com/influxdata/telegraf/plugins/inputs/leofs"
	_ "github.com/influxdata/telegraf/plugins/inputs/logparser"
	_ "github.com/influxdata/telegraf/plugins/inputs/lustre2"
	_ "github.com/influxdata/telegraf/plugins/inputs/mailchimp"
	_ "github.com/influxdata/telegraf/plugins/inputs/memcached"
	_ "github.com/influxdata/telegraf/plugins/inputs/mesos"
	_ "github.com/influxdata/telegraf/plugins/inputs/mongodb"
	_ "github.com/influxdata/telegraf/plugins/inputs/mqtt_consumer"
	_ "github.com/influxdata/telegraf/plugins/inputs/mysql"
	_ "github.com/influxdata/telegraf/plugins/inputs/nats_consumer"
	_ "github.com/influxdata/telegraf/plugins/inputs/net_response"
	_ "github.com/influxdata/telegraf/plugins/inputs/nginx"
	_ "github.com/influxdata/telegraf/plugins/inputs/nsq"
	_ "github.com/influxdata/telegraf/plugins/inputs/nsq_consumer"
	_ "github.com/influxdata/telegraf/plugins/inputs/nstat"
	_ "github.com/influxdata/telegraf/plugins/inputs/ntpq"
	_ "github.com/influxdata/telegraf/plugins/inputs/passenger"
	_ "github.com/influxdata/telegraf/plugins/inputs/phpfpm"
	_ "github.com/influxdata/telegraf/plugins/inputs/ping"
	_ "github.com/influxdata/telegraf/plugins/inputs/postgresql"
	_ "github.com/influxdata/telegraf/plugins/inputs/postgresql_extensible"
	_ "github.com/influxdata/telegraf/plugins/inputs/powerdns"
	_ "github.com/influxdata/telegraf/plugins/inputs/procstat"
	_ "github.com/influxdata/telegraf/plugins/inputs/prometheus"
	_ "github.com/influxdata/telegraf/plugins/inputs/puppetagent"
	_ "github.com/influxdata/telegraf/plugins/inputs/rabbitmq"
	_ "github.com/influxdata/telegraf/plugins/inputs/raindrops"
	_ "github.com/influxdata/telegraf/plugins/inputs/redis"
	_ "github.com/influxdata/telegraf/plugins/inputs/rethinkdb"
	_ "github.com/influxdata/telegraf/plugins/inputs/riak"
	_ "github.com/influxdata/telegraf/plugins/inputs/sensors"
	_ "github.com/influxdata/telegraf/plugins/inputs/snmp"
	_ "github.com/influxdata/telegraf/plugins/inputs/sqlserver"
	_ "github.com/influxdata/telegraf/plugins/inputs/statsd"
	_ "github.com/influxdata/telegraf/plugins/inputs/sysstat"
	_ "github.com/influxdata/telegraf/plugins/inputs/system"
	_ "github.com/influxdata/telegraf/plugins/inputs/tail"
	_ "github.com/influxdata/telegraf/plugins/inputs/tcp_listener"
	_ "github.com/influxdata/telegraf/plugins/inputs/trig"
	_ "github.com/influxdata/telegraf/plugins/inputs/twemproxy"
	_ "github.com/influxdata/telegraf/plugins/inputs/udp_listener"
	_ "github.com/influxdata/telegraf/plugins/inputs/varnish"
	_ "github.com/influxdata/telegraf/plugins/inputs/webhooks"
	_ "github.com/influxdata/telegraf/plugins/inputs/win_perf_counters"
	_ "github.com/influxdata/telegraf/plugins/inputs/zfs"
	_ "github.com/influxdata/telegraf/plugins/inputs/zookeeper"
)
<|endoftext|>


<commit_before>package test

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/fatih/color"
	"github.com/ghodss/yaml"
	"github.com/servehub/serve/manifest"
	"github.com/servehub/utils"
	"github.com/servehub/utils/mergemap"
)

func init() {
	manifest.PluginRegestry.Add("test.component", TestComponent{})
}

type TestComponent struct{}

func (p TestComponent) Run(data manifest.Manifest) error {
	if data.GetString("env") != data.GetString("current-env") {
		log.Printf("No component test found for `%s`.\n", data.GetString("current-env"))
		return nil
	}

	for i, multiComponent := range data.GetArray("components") {
		component := manifest.ParseJSON(data.GetTree("compose.services.component").String())
		merged, _ := mergemap.Merge(component.Unwrap().(map[string]interface{}), multiComponent.Unwrap().(map[string]interface{}))
		name := fmt.Sprintf("component-%d", i+2)

		data.Set("compose.services."+name, merged)
		data.ArrayAppend("compose.services.tests.depends_on", name)
	}

	bytes, finalErr := yaml.Marshal(data.GetTree("compose").Unwrap())
	if finalErr != nil {
		return fmt.Errorf("error on serialize yaml: %v", finalErr)
	}

	tmpfile, finalErr := ioutil.TempFile("", "serve-component-test")
	if finalErr != nil {
		return fmt.Errorf("error create tmpfile: %v", finalErr)
	}

	defer func() {
		tmpfile.Close()
		os.Remove(tmpfile.Name())
	}()

	if _, err := tmpfile.Write(bytes); err != nil {
		return fmt.Errorf("error write to tmpfile: %v", err)
	}

	checkFile := data.GetStringOr("check-file-exist", "")
	if checkFile != "" {
		os.Remove(checkFile)
	}

	if err := utils.RunCmd("docker-compose -p %s -f %s pull", data.GetString("name"), tmpfile.Name()); err != nil {
		return fmt.Errorf("error on pull new images for docker-compose: %v", err)
	}

	defer func() {
		utils.RunCmd("docker-compose -p %s -f %s down -v --remove-orphans", data.GetString("name"), tmpfile.Name())
	}()

	timeout, finalErr := time.ParseDuration(data.GetString("timeout"))
	if finalErr != nil {
		return fmt.Errorf("error parse timeout: %v", finalErr)
	}

	go func() {
		select {
		case <-time.After(timeout):
			color.Red("Timeout exceeded for tests, exit...")
			utils.RunCmd("docker-compose -p %s -f %s down -v --remove-orphans", data.GetString("name"), tmpfile.Name())
		}
	}()

	if res := utils.RunCmd("DOCKER_CLIENT_TIMEOUT=300 COMPOSE_HTTP_TIMEOUT=300 docker-compose -p %s -f %s up --abort-on-container-exit", data.GetString("name"), tmpfile.Name()); res != nil {
		finalErr = fmt.Errorf("error on running docker-compose with tests: %s", res)
	}

	if checkFile != "" && finalErr == nil {
		log.Printf("Check if %s exist...", checkFile)

		if info, err := os.Stat(checkFile); os.IsNotExist(err) || info.Size() < 16 {
			finalErr = fmt.Errorf("check file not exist! %s. %s", checkFile, err)
		}
	}

	return finalErr
}
<commit_msg>code style<commit_after>package test

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/fatih/color"
	"github.com/ghodss/yaml"
	"github.com/servehub/serve/manifest"
	"github.com/servehub/utils"
	"github.com/servehub/utils/mergemap"
)

func init() {
	manifest.PluginRegestry.Add("test.component", TestComponent{})
}

type TestComponent struct{}

func (p TestComponent) Run(data manifest.Manifest) error {
	if data.GetString("env") != data.GetString("current-env") {
		log.Printf("No component test found for `%s`.\n", data.GetString("current-env"))
		return nil
	}

	for i, multiComponent := range data.GetArray("components") {
		component := manifest.ParseJSON(data.GetTree("compose.services.component").String())
		merged, _ := mergemap.Merge(component.Unwrap().(map[string]interface{}), multiComponent.Unwrap().(map[string]interface{}))
		name := fmt.Sprintf("component-%d", i+2)

		data.Set("compose.services."+name, merged)
		data.ArrayAppend("compose.services.tests.depends_on", name)
	}

	bytes, finalErr := yaml.Marshal(data.GetTree("compose").Unwrap())
	if finalErr != nil {
		return fmt.Errorf("error on serialize yaml: %v", finalErr)
	}

	tmpfile, finalErr := ioutil.TempFile("", "serve-component-test")
	if finalErr != nil {
		return fmt.Errorf("error create tmpfile: %v", finalErr)
	}

	defer func() {
		tmpfile.Close()
		os.Remove(tmpfile.Name())
	}()

	if _, err := tmpfile.Write(bytes); err != nil {
		return fmt.Errorf("error write to tmpfile: %v", err)
	}

	checkFile := data.GetStringOr("check-file-exist", "")
	if checkFile != "" {
		os.Remove(checkFile)
	}

	if err := utils.RunCmd("docker-compose -p %s -f %s pull", data.GetString("name"), tmpfile.Name()); err != nil {
		return fmt.Errorf("error on pull new images for docker-compose: %v", err)
	}

	defer func() {
		utils.RunCmd("docker-compose -p %s -f %s down -v --remove-orphans", data.GetString("name"), tmpfile.Name())
	}()

	timeout, finalErr := time.ParseDuration(data.GetString("timeout"))
	if finalErr != nil {
		return fmt.Errorf("error parse timeout: %v", finalErr)
	}

	go func() {
		select {
		case <-time.After(timeout):
			color.Red("Timeout exceeded for tests, exit...")
			utils.RunCmd("docker-compose -p %s -f %s down -v --remove-orphans", data.GetString("name"), tmpfile.Name())
		}
	}()

	if res := utils.RunCmd("DOCKER_CLIENT_TIMEOUT=300 COMPOSE_HTTP_TIMEOUT=300 docker-compose -p %s -f %s up --abort-on-container-exit", data.GetString("name"), tmpfile.Name()); res != nil {
		finalErr = fmt.Errorf("error on running docker-compose with tests: %s", res)
	}

	if checkFile != "" && finalErr == nil {
		log.Printf("Check if %s exist...", checkFile)

		if info, err := os.Stat(checkFile); os.IsNotExist(err) || info.Size() < 16 {
			finalErr = fmt.Errorf("check file not exist! %s. %v", checkFile, err)
		}
	}

	return finalErr
}
<|endoftext|>


<commit_before>package fcm

type Client interface {
	// Sends FCM message to FCM server.
	Send(notification Notification, data Data, regIds ...string) (Response, error)
}

type Notification struct {
	Title       string
	Body        string
	Icon        string
	Sound       string
	Badge       string
	Tag         string
	Color       string
	ClickAction string
}

type Data map[string]interface{}
<commit_msg>add Response<commit_after>package fcm

type Client interface {
	// Sends FCM message to FCM server.
	Send(notification Notification, data Data, regIds ...string) (Response, error)
}

type Notification struct {
	Title       string
	Body        string
	Icon        string
	Sound       string
	Badge       string
	Tag         string
	Color       string
	ClickAction string
}

type Data map[string]interface{}

type Response struct {
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Jennal(jennalcn@gmail.com). All rights reserved.
//
// Licensed under the MIT License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// http://opensource.org/licenses/MIT
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

package service

import (
	"reflect"
	"sync"
	"time"

	"fmt"

	"github.com/jennal/goplay/aop"
	"github.com/jennal/goplay/encode"
	"github.com/jennal/goplay/filter"
	"github.com/jennal/goplay/helpers"
	"github.com/jennal/goplay/log"
	"github.com/jennal/goplay/pkg"
	"github.com/jennal/goplay/router"
	"github.com/jennal/goplay/session"
	"github.com/jennal/goplay/transfer"
)

const (
	REQUEST_TIMEOUT = 3 * time.Second
)

type requestCallbacks struct {
	startTime      time.Time
	successCallbak *Method
	failCallback   *Method
}

type ServiceClient struct {
	*session.Session

	router  *router.Router
	filters []filter.IFilter

	requestCbsMutex sync.Mutex
	requestCbs      map[pkg.PackageIDType]*requestCallbacks

	pushCbsMutex sync.Mutex
	pushCbs      map[string][]*Method
}

func NewServiceClient(cli transfer.IClient) *ServiceClient {
	result := &ServiceClient{
		Session: session.NewSession(cli),

		router:  nil,
		filters: nil,

		requestCbs: make(map[pkg.PackageIDType]*requestCallbacks),
		pushCbs:    make(map[string][]*Method),
	}
	result.setupEventLoop()

	return result
}

func (self *ServiceClient) SetRouter(router *router.Router) {
	self.router = router
}

func (self *ServiceClient) SetFilters(filters []filter.IFilter) {
	self.filters = filters
}

func (s *ServiceClient) checkTimeoutLoop() {
	for {
		if !s.IsConnected() {
			break
		}

		ids := []pkg.PackageIDType{}

		s.requestCbsMutex.Lock()
		for id, item := range s.requestCbs {
			if time.Since(item.startTime) > REQUEST_TIMEOUT {
				ids = append(ids, id)
				item.failCallback.Call(pkg.NewErrorMessage(pkg.STAT_ERR_TIMEOUT, "Request Timeout"))
			}
		}

		for _, id := range ids {
			delete(s.requestCbs, id)
		}
		s.requestCbsMutex.Unlock()

		time.Sleep(REQUEST_TIMEOUT)
	}
}

func (s *ServiceClient) setupEventLoop() {
	var exitChan chan int
	s.On(transfer.EVENT_CLIENT_CONNECTED, s, func(client transfer.IClient) {
		sess := s.Session

		if s.filters != nil && len(s.filters) > 0 {
			for _, filter := range s.filters {
				if !filter.OnNewClient(sess) {
					return
				}
			}
		}

		go s.checkTimeoutLoop()
		go func() {
			aop.Recover(func() {
			Loop:
				for {
					select {
					case <-exitChan:
						break Loop
					default:
						header, bodyBuf, err := sess.Recv()
						if err != nil {
							log.Errorf("Recv:\n\terr => %v\n\theader => %#v\n\tbody => %#v | %v", err, header, bodyBuf, string(bodyBuf))
							sess.Disconnect()
							break Loop
						}

						if header.Type != pkg.PKG_HEARTBEAT && header.Type != pkg.PKG_HEARTBEAT_RESPONSE {
							log.Logf("Recv:\n\theader => %#v\n\tbody => %#v | %v\n\terr => %v\n", header, bodyBuf, string(bodyBuf), err)
						}

						//filters
						if s.filters != nil && len(s.filters) > 0 {
							for _, filter := range s.filters {
								if !filter.OnRecv(sess, header, bodyBuf) {
									goto Loop
								}
							}
						}

						switch header.Type {
						case pkg.PKG_REQUEST, pkg.PKG_RPC_REQUEST:
							if s.router != nil {
								results, err := s.callRouteFunc(sess, header, bodyBuf)
								if err != nil {
									log.Errorf("CallRouteFunc:\n\terr => %v\n\theader => %#v\n\tbody => %#v | %v", err, header, bodyBuf, string(bodyBuf))
									sess.Disconnect()
									break Loop
								}
								// fmt.Printf(" => Loop result: %#v\n", results)
								err = s.response(sess, header, results)
								if err != nil {
									log.Errorf("Response:\n\terr => %v\n\theader => %#v\n\tresults => %#v", err, header, results)
									sess.Disconnect()
									break Loop
								}
							}
						case pkg.PKG_NOTIFY, pkg.PKG_RPC_NOTIFY:
							if s.router != nil {
								_, err := s.callRouteFunc(sess, header, bodyBuf)
								if err != nil {
									log.Errorf("CallRouteFunc:\n\terr => %v\n\theader => %#v\n\tbody => %#v | %v", err, header, bodyBuf, string(bodyBuf))
									sess.Disconnect()
									break Loop
								}
							}
						case pkg.PKG_PUSH, pkg.PKG_RPC_PUSH:
							s.recvPush(header, bodyBuf)
						case pkg.PKG_RESPONSE, pkg.PKG_RPC_RESPONSE:
							s.recvResponse(header, bodyBuf)
						case pkg.PKG_HEARTBEAT, pkg.PKG_HEARTBEAT_RESPONSE:
							fallthrough
						default:
							log.Errorf("Can't reach here!!\n\terr => %v\n\theader => %#v\n\tbody => %#v", err, header, bodyBuf)
							break
						}
					}
				}
			}, func(err interface{}) {
				if err != nil && err.(error) != nil {
					log.Error(err.(error))
				}

				if sess.IsConnected() {
					sess.Disconnect()
				}
			})
		}()
	})
	s.On(transfer.EVENT_CLIENT_DISCONNECTED, s, func(cli transfer.IClient) {
		exitChan <- 1
	})
}

func (s *ServiceClient) callRouteFunc(sess *session.Session, header *pkg.Header, bodyBuf []byte) ([]interface{}, error) {
	/*
	 * 1. find route func
	 * 2. unmarshal data
	 * 3. call route func
	 */
	method := s.router.Get(header.Route)
	if method == nil {
		return nil, log.NewErrorf("Can't find method with route: %s", header.Route)
	}
	val := method.NewArg(2)
	// fmt.Printf("Service.callRouteFunc: %#v => %v\n", val, reflect.TypeOf(val))
	decoder := encode.GetEncodeDecoder(header.Encoding)
	err := decoder.Unmarshal(bodyBuf, val)
	if err != nil {
		return nil, log.NewErrorf("Service.callRouteFunc decoder.Unmarshal failed: %v", err)
	}
	// fmt.Printf("Service.callRouteFunc: %#v => %v\n", val, reflect.TypeOf(val))

	var result []interface{}
	aop.Recover(func() {
		result = method.Call(sess, helpers.GetValueFromPtr(val))
	}, func(e interface{}) {
		err = e.(error)
	})

	return result, err
}

func (s *ServiceClient) response(sess *session.Session, header *pkg.Header, results []interface{}) error {
	respHeader := *header
	if header.Type == pkg.PKG_RPC_REQUEST {
		respHeader.Type = pkg.PKG_RPC_RESPONSE
	} else {
		respHeader.Type = pkg.PKG_RESPONSE
	}

	if results == nil || len(results) <= 0 {
		return sess.Send(&respHeader, []byte{})
	}

	result := results[0]
	/* check error != nil */
	if len(results) == 2 && !reflect.ValueOf(results[1]).IsNil() {
		respHeader.Status = pkg.STAT_ERR
		result = results[1]
	}

	// fmt.Println("result:", result)

	encoder := encode.GetEncodeDecoder(header.Encoding)
	body, err := encoder.Marshal(result)
	if err != nil {
		return err
	}

	return sess.Send(&respHeader, body)
}

func (s *ServiceClient) recvPush(header *pkg.Header, body []byte) {
	s.pushCbsMutex.Lock()
	list, ok := s.pushCbs[header.Route]
	s.pushCbsMutex.Unlock()

	if !ok {
		return
	}

	for _, item := range list {
		val := item.NewArg(0)
		s.Encoder.Unmarshal(body, val)
		item.Call(helpers.GetValueFromPtr(val))
	}
}

func (s *ServiceClient) recvResponse(header *pkg.Header, body []byte) {
	s.requestCbsMutex.Lock()
	cbs, ok := s.requestCbs[header.ID]
	if ok {
		delete(s.requestCbs, header.ID)
	}
	s.requestCbsMutex.Unlock()

	if !ok {
		return
	}

	// log.Logf("%v %v %v", header.Status, body, string(body))
	if header.Status == pkg.STAT_OK {
		val := cbs.successCallbak.NewArg(0)
		err := s.Encoder.Unmarshal(body, val)
		if err == nil {
			cbs.successCallbak.Call(helpers.GetValueFromPtr(val))
			return
		}
	} else {
		val := cbs.failCallback.NewArg(0)
		err := s.Encoder.Unmarshal(body, val)
		if err == nil {
			cbs.failCallback.Call(helpers.GetValueFromPtr(val))
			return
		}
	}

	cbs.failCallback.Call(pkg.NewErrorMessage(
		pkg.STAT_ERR_DECODE_FAILED,
		fmt.Sprintf("decode body failed: %#v | %v", body, string(body))))
}

func (s *ServiceClient) Request(route string, data interface{}, succCb interface{}, failCb func(*pkg.ErrorMessage)) error {
	header := s.NewHeader(pkg.PKG_RPC_REQUEST, s.Encoding, route)
	cbs := requestCallbacks{
		successCallbak: NewMethod(succCb),
		failCallback:   NewMethod(failCb),
		startTime:      time.Now(),
	}

	s.requestCbsMutex.Lock()
	s.requestCbs[header.ID] = &cbs
	s.requestCbsMutex.Unlock()

	buf, err := s.Encoder.Marshal(data)
	if err != nil {
		return err
	}
	return s.Send(header, buf)
}

func (s *ServiceClient) Notify(route string, data interface{}) error {
	header := s.NewHeader(pkg.PKG_RPC_NOTIFY, s.Encoding, route)
	buf, err := s.Encoder.Marshal(data)
	if err != nil {
		return err
	}
	return s.Send(header, buf)
}

func (s *ServiceClient) Push(route string, data interface{}) error {
	header := s.NewHeader(pkg.PKG_RPC_PUSH, s.Encoding, route)
	buf, err := s.Encoder.Marshal(data)
	if err != nil {
		return err
	}
	return s.Send(header, buf)
}

func (s *ServiceClient) AddListener(route string, callback interface{}) {
	s.pushCbsMutex.Lock()
	defer s.pushCbsMutex.Unlock()

	list, ok := s.pushCbs[route]
	if !ok {
		list = make([]*Method, 0)
		s.pushCbs[route] = list
	}

	s.pushCbs[route] = append(s.pushCbs[route], NewMethod(callback))
}
<commit_msg>make heartbeat filter default to service client<commit_after>// Copyright (C) 2017 Jennal(jennalcn@gmail.com). All rights reserved.
//
// Licensed under the MIT License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
//
// http://opensource.org/licenses/MIT
//
// Unless required by applicable law or agreed to in writing, software distributed
// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.

package service

import (
	"reflect"
	"sync"
	"time"

	"fmt"

	"github.com/jennal/goplay/aop"
	"github.com/jennal/goplay/encode"
	"github.com/jennal/goplay/filter"
	"github.com/jennal/goplay/filter/heartbeat"
	"github.com/jennal/goplay/helpers"
	"github.com/jennal/goplay/log"
	"github.com/jennal/goplay/pkg"
	"github.com/jennal/goplay/router"
	"github.com/jennal/goplay/session"
	"github.com/jennal/goplay/transfer"
)

const (
	REQUEST_TIMEOUT = 3 * time.Second
)

type requestCallbacks struct {
	startTime      time.Time
	successCallbak *Method
	failCallback   *Method
}

type ServiceClient struct {
	*session.Session

	router  *router.Router
	filters []filter.IFilter

	requestCbsMutex sync.Mutex
	requestCbs      map[pkg.PackageIDType]*requestCallbacks

	pushCbsMutex sync.Mutex
	pushCbs      map[string][]*Method
}

func NewServiceClient(cli transfer.IClient) *ServiceClient {
	result := &ServiceClient{
		Session: session.NewSession(cli),

		router: nil,
		filters: []filter.IFilter{
			heartbeat.NewHeartBeatManager(),
		},

		requestCbs: make(map[pkg.PackageIDType]*requestCallbacks),
		pushCbs:    make(map[string][]*Method),
	}
	result.setupEventLoop()

	return result
}

func (self *ServiceClient) SetRouter(router *router.Router) {
	self.router = router
}

func (self *ServiceClient) SetFilters(filters []filter.IFilter) {
	self.filters = filters
}

func (s *ServiceClient) checkTimeoutLoop() {
	for {
		if !s.IsConnected() {
			break
		}

		ids := []pkg.PackageIDType{}

		s.requestCbsMutex.Lock()
		for id, item := range s.requestCbs {
			if time.Since(item.startTime) > REQUEST_TIMEOUT {
				ids = append(ids, id)
				item.failCallback.Call(pkg.NewErrorMessage(pkg.STAT_ERR_TIMEOUT, "Request Timeout"))
			}
		}

		for _, id := range ids {
			delete(s.requestCbs, id)
		}
		s.requestCbsMutex.Unlock()

		time.Sleep(REQUEST_TIMEOUT)
	}
}

func (s *ServiceClient) setupEventLoop() {
	var exitChan chan int
	s.On(transfer.EVENT_CLIENT_CONNECTED, s, func(client transfer.IClient) {
		sess := s.Session

		if s.filters != nil && len(s.filters) > 0 {
			for _, filter := range s.filters {
				if !filter.OnNewClient(sess) {
					return
				}
			}
		}

		go s.checkTimeoutLoop()
		go func() {
			aop.Recover(func() {
			Loop:
				for {
					select {
					case <-exitChan:
						break Loop
					default:
						header, bodyBuf, err := sess.Recv()
						if err != nil {
							log.Errorf("Recv:\n\terr => %v\n\theader => %#v\n\tbody => %#v | %v", err, header, bodyBuf, string(bodyBuf))
							sess.Disconnect()
							break Loop
						}

						if header.Type != pkg.PKG_HEARTBEAT && header.Type != pkg.PKG_HEARTBEAT_RESPONSE {
							log.Logf("Recv:\n\theader => %#v\n\tbody => %#v | %v\n\terr => %v\n", header, bodyBuf, string(bodyBuf), err)
						}

						//filters
						if s.filters != nil && len(s.filters) > 0 {
							for _, filter := range s.filters {
								if !filter.OnRecv(sess, header, bodyBuf) {
									goto Loop
								}
							}
						}

						switch header.Type {
						case pkg.PKG_REQUEST, pkg.PKG_RPC_REQUEST:
							if s.router != nil {
								results, err := s.callRouteFunc(sess, header, bodyBuf)
								if err != nil {
									log.Errorf("CallRouteFunc:\n\terr => %v\n\theader => %#v\n\tbody => %#v | %v", err, header, bodyBuf, string(bodyBuf))
									sess.Disconnect()
									break Loop
								}
								// fmt.Printf(" => Loop result: %#v\n", results)
								err = s.response(sess, header, results)
								if err != nil {
									log.Errorf("Response:\n\terr => %v\n\theader => %#v\n\tresults => %#v", err, header, results)
									sess.Disconnect()
									break Loop
								}
							}
						case pkg.PKG_NOTIFY, pkg.PKG_RPC_NOTIFY:
							if s.router != nil {
								_, err := s.callRouteFunc(sess, header, bodyBuf)
								if err != nil {
									log.Errorf("CallRouteFunc:\n\terr => %v\n\theader => %#v\n\tbody => %#v | %v", err, header, bodyBuf, string(bodyBuf))
									sess.Disconnect()
									break Loop
								}
							}
						case pkg.PKG_PUSH, pkg.PKG_RPC_PUSH:
							s.recvPush(header, bodyBuf)
						case pkg.PKG_RESPONSE, pkg.PKG_RPC_RESPONSE:
							s.recvResponse(header, bodyBuf)
						case pkg.PKG_HEARTBEAT, pkg.PKG_HEARTBEAT_RESPONSE:
							fallthrough
						default:
							log.Errorf("Can't reach here!!\n\terr => %v\n\theader => %#v\n\tbody => %#v", err, header, bodyBuf)
							break
						}
					}
				}
			}, func(err interface{}) {
				if err != nil && err.(error) != nil {
					log.Error(err.(error))
				}

				if sess.IsConnected() {
					sess.Disconnect()
				}
			})
		}()
	})
	s.On(transfer.EVENT_CLIENT_DISCONNECTED, s, func(cli transfer.IClient) {
		exitChan <- 1
	})
}

func (s *ServiceClient) callRouteFunc(sess *session.Session, header *pkg.Header, bodyBuf []byte) ([]interface{}, error) {
	/*
	 * 1. find route func
	 * 2. unmarshal data
	 * 3. call route func
	 */
	method := s.router.Get(header.Route)
	if method == nil {
		return nil, log.NewErrorf("Can't find method with route: %s", header.Route)
	}
	val := method.NewArg(2)
	// fmt.Printf("Service.callRouteFunc: %#v => %v\n", val, reflect.TypeOf(val))
	decoder := encode.GetEncodeDecoder(header.Encoding)
	err := decoder.Unmarshal(bodyBuf, val)
	if err != nil {
		return nil, log.NewErrorf("Service.callRouteFunc decoder.Unmarshal failed: %v", err)
	}
	// fmt.Printf("Service.callRouteFunc: %#v => %v\n", val, reflect.TypeOf(val))

	var result []interface{}
	aop.Recover(func() {
		result = method.Call(sess, helpers.GetValueFromPtr(val))
	}, func(e interface{}) {
		err = e.(error)
	})

	return result, err
}

func (s *ServiceClient) response(sess *session.Session, header *pkg.Header, results []interface{}) error {
	respHeader := *header
	if header.Type == pkg.PKG_RPC_REQUEST {
		respHeader.Type = pkg.PKG_RPC_RESPONSE
	} else {
		respHeader.Type = pkg.PKG_RESPONSE
	}

	if results == nil || len(results) <= 0 {
		return sess.Send(&respHeader, []byte{})
	}

	result := results[0]
	/* check error != nil */
	if len(results) == 2 && !reflect.ValueOf(results[1]).IsNil() {
		respHeader.Status = pkg.STAT_ERR
		result = results[1]
	}

	// fmt.Println("result:", result)

	encoder := encode.GetEncodeDecoder(header.Encoding)
	body, err := encoder.Marshal(result)
	if err != nil {
		return err
	}

	return sess.Send(&respHeader, body)
}

func (s *ServiceClient) recvPush(header *pkg.Header, body []byte) {
	s.pushCbsMutex.Lock()
	list, ok := s.pushCbs[header.Route]
	s.pushCbsMutex.Unlock()

	if !ok {
		return
	}

	for _, item := range list {
		val := item.NewArg(0)
		s.Encoder.Unmarshal(body, val)
		item.Call(helpers.GetValueFromPtr(val))
	}
}

func (s *ServiceClient) recvResponse(header *pkg.Header, body []byte) {
	s.requestCbsMutex.Lock()
	cbs, ok := s.requestCbs[header.ID]
	if ok {
		delete(s.requestCbs, header.ID)
	}
	s.requestCbsMutex.Unlock()

	if !ok {
		return
	}

	// log.Logf("%v %v %v", header.Status, body, string(body))
	if header.Status == pkg.STAT_OK {
		val := cbs.successCallbak.NewArg(0)
		err := s.Encoder.Unmarshal(body, val)
		if err == nil {
			cbs.successCallbak.Call(helpers.GetValueFromPtr(val))
			return
		}
	} else {
		val := cbs.failCallback.NewArg(0)
		err := s.Encoder.Unmarshal(body, val)
		if err == nil {
			cbs.failCallback.Call(helpers.GetValueFromPtr(val))
			return
		}
	}

	cbs.failCallback.Call(pkg.NewErrorMessage(
		pkg.STAT_ERR_DECODE_FAILED,
		fmt.Sprintf("decode body failed: %#v | %v", body, string(body))))
}

func (s *ServiceClient) Request(route string, data interface{}, succCb interface{}, failCb func(*pkg.ErrorMessage)) error {
	header := s.NewHeader(pkg.PKG_RPC_REQUEST, s.Encoding, route)
	cbs := requestCallbacks{
		successCallbak: NewMethod(succCb),
		failCallback:   NewMethod(failCb),
		startTime:      time.Now(),
	}

	s.requestCbsMutex.Lock()
	s.requestCbs[header.ID] = &cbs
	s.requestCbsMutex.Unlock()

	buf, err := s.Encoder.Marshal(data)
	if err != nil {
		return err
	}
	return s.Send(header, buf)
}

func (s *ServiceClient) Notify(route string, data interface{}) error {
	header := s.NewHeader(pkg.PKG_RPC_NOTIFY, s.Encoding, route)
	buf, err := s.Encoder.Marshal(data)
	if err != nil {
		return err
	}
	return s.Send(header, buf)
}

func (s *ServiceClient) Push(route string, data interface{}) error {
	header := s.NewHeader(pkg.PKG_RPC_PUSH, s.Encoding, route)
	buf, err := s.Encoder.Marshal(data)
	if err != nil {
		return err
	}
	return s.Send(header, buf)
}

func (s *ServiceClient) AddListener(route string, callback interface{}) {
	s.pushCbsMutex.Lock()
	defer s.pushCbsMutex.Unlock()

	list, ok := s.pushCbs[route]
	if !ok {
		list = make([]*Method, 0)
		s.pushCbs[route] = list
	}

	s.pushCbs[route] = append(s.pushCbs[route], NewMethod(callback))
}
<|endoftext|>


<commit_before>// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"math"
	"net/http"
	"strconv"
	"time"

	"github.com/goharbor/harbor/src/common/dao"
	common_http "github.com/goharbor/harbor/src/common/http"
	common_job "github.com/goharbor/harbor/src/common/job"
	common_models "github.com/goharbor/harbor/src/common/models"
	"github.com/goharbor/harbor/src/controller/scan"
	"github.com/goharbor/harbor/src/core/api/models"
	utils_core "github.com/goharbor/harbor/src/core/utils"
	"github.com/goharbor/harbor/src/lib/errors"
	"github.com/goharbor/harbor/src/lib/log"
)

// AJAPI manages the CRUD of admin job and its schedule, any API wants to handle manual and cron job like ScanAll and GC cloud reuse it.
type AJAPI struct {
	BaseController
}

// Prepare validates the URL and parms, it needs the system admin permission.
func (aj *AJAPI) Prepare() {
	aj.BaseController.Prepare()
}

// updateSchedule update a schedule of admin job.
func (aj *AJAPI) updateSchedule(ajr models.AdminJobReq) {
	if ajr.Schedule.Type == models.ScheduleManual {
		aj.SendInternalServerError((fmt.Errorf("fail to update admin job schedule as wrong schedule type: %s", ajr.Schedule.Type)))
		return
	}

	query := &common_models.AdminJobQuery{
		Name: ajr.Name,
		Kind: common_job.JobKindPeriodic,
	}
	jobs, err := dao.GetAdminJobs(query)
	if err != nil {
		aj.SendInternalServerError(err)
		return
	}
	if len(jobs) != 1 {
		aj.SendInternalServerError(errors.New("fail to update admin job schedule as we found more than one schedule in system, please ensure that only one schedule left for your job"))
		return
	}

	// stop the scheduled job and remove it.
	if err = utils_core.GetJobServiceClient().PostAction(jobs[0].UUID, common_job.JobActionStop); err != nil {
		_, ok := err.(*common_job.StatusBehindError)
		if !ok {
			if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound {
				aj.SendInternalServerError(err)
				return
			}
		}
	}

	if err = dao.DeleteAdminJob(jobs[0].ID); err != nil {
		aj.SendInternalServerError(err)
		return
	}

	// Set schedule to None means to cancel the schedule, won't add new job.
	if ajr.Schedule.Type != models.ScheduleNone {
		aj.submit(&ajr)
	}
}

// get get a execution of admin job by ID
func (aj *AJAPI) get(id int64) {
	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
		ID: id,
	})
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("failed to get admin jobs: %v", err))
		return
	}
	if len(jobs) == 0 {
		aj.SendNotFoundError(errors.New("no admin job found"))
		return
	}

	adminJobRep, err := convertToAdminJobRep(jobs[0])
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("failed to convert admin job response: %v", err))
		return
	}

	aj.Data["json"] = adminJobRep
	aj.ServeJSON()
}

// list list all executions of admin job by name
func (aj *AJAPI) list(name string) {
	jobs, err := dao.GetTop10AdminJobsOfName(name)
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("failed to get admin jobs: %v", err))
		return
	}

	AdminJobReps := []*models.AdminJobRep{}
	for _, job := range jobs {
		AdminJobRep, err := convertToAdminJobRep(job)
		if err != nil {
			aj.SendInternalServerError(fmt.Errorf("failed to convert admin job response: %v", err))
			return
		}
		AdminJobReps = append(AdminJobReps, &AdminJobRep)
	}

	aj.Data["json"] = AdminJobReps
	aj.ServeJSON()
}

// getSchedule gets admin job schedule ...
func (aj *AJAPI) getSchedule(name string) {
	result := models.AdminJobRep{}

	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
		Name: name,
		Kind: common_job.JobKindPeriodic,
	})
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("failed to get admin jobs: %v", err))
		return
	}
	if len(jobs) > 1 {
		aj.SendInternalServerError(errors.New("get more than one scheduled admin job, make sure there has only one"))
		return
	}

	if len(jobs) != 0 {
		adminJobRep, err := convertToAdminJobRep(jobs[0])
		if err != nil {
			aj.SendInternalServerError(fmt.Errorf("failed to convert admin job response: %v", err))
			return
		}
		result.Schedule = adminJobRep.Schedule
		result.Parameters = adminJobRep.Parameters
	}

	aj.Data["json"] = result
	aj.ServeJSON()
}

// getLog ...
func (aj *AJAPI) getLog(id int64) {
	job, err := dao.GetAdminJob(id)
	if err != nil {
		log.Errorf("Failed to load job data for job: %d, error: %v", id, err)
		aj.SendInternalServerError(errors.New("Failed to get Job data"))
		return
	}
	if job == nil {
		log.Errorf("Failed to get admin job: %d", id)
		aj.SendNotFoundError(errors.New("Failed to get Job"))
		return
	}

	var jobID string
	// to get the latest execution job id, then to query job log.
	if job.Kind == common_job.JobKindPeriodic {
		exes, err := utils_core.GetJobServiceClient().GetExecutions(job.UUID)
		if err != nil {
			aj.SendInternalServerError(err)
			return
		}
		if len(exes) == 0 {
			aj.SendNotFoundError(errors.New("no execution log found"))
			return
		}
		// get the latest terminal status execution.
		for _, exe := range exes {
			if exe.Info.Status == "Error" || exe.Info.Status == "Success" {
				jobID = exe.Info.JobID
				break
			}
		}
		// no execution found
		if jobID == "" {
			aj.SendNotFoundError(errors.New("no execution log found"))
			return
		}

	} else {
		jobID = job.UUID
	}

	logBytes, err := utils_core.GetJobServiceClient().GetJobLog(jobID)
	if err != nil {
		if httpErr, ok := err.(*common_http.Error); ok {
			aj.RenderError(httpErr.Code, "")
			log.Errorf(fmt.Sprintf("failed to get log of job %d: %d %s",
				id, httpErr.Code, httpErr.Message))
			return
		}
		aj.SendInternalServerError(fmt.Errorf("Failed to get job logs, uuid: %s, error: %v", job.UUID, err))
		return
	}
	aj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
	aj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
	_, err = aj.Ctx.ResponseWriter.Write(logBytes)
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("Failed to write job logs, uuid: %s, error: %v", job.UUID, err))
	}
}

// submit submits a job to job service per request
func (aj *AJAPI) submit(ajr *models.AdminJobReq) {
	// when the schedule is saved as None without any schedule, just return 200 and do nothing.
	if ajr.Schedule == nil || ajr.Schedule.Type == models.ScheduleNone {
		return
	}

	// cannot post multiple schedule for admin job.
	if ajr.IsPeriodic() {
		jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
			Name: ajr.Name,
			Kind: common_job.JobKindPeriodic,
		})
		if err != nil {
			aj.SendInternalServerError(fmt.Errorf("failed to get admin jobs: %v", err))
			return
		}
		if len(jobs) != 0 {
			aj.SendPreconditionFailedError(errors.New("fail to set schedule for admin job as always had one, please delete it firstly then to re-schedule"))
			return
		}
	} else {
		// So far, it should be a generic job for the manually trigger case.
		// Only needs to care the 1st generic job.
		// Check if there are still ongoing scan jobs triggered by the previous admin job.
		// TODO: REPLACE WITH TASK MANAGER METHODS IN FUTURE
		jb, err := aj.getLatestAdminJob(ajr.Name, common_job.JobKindGeneric)
		if err != nil {
			aj.SendInternalServerError(errors.Wrap(err, "AJAPI"))
			return
		}

		if jb != nil {
			// With a reasonable timeout duration
			if jb.UpdateTime.Add(2 * time.Hour).After(time.Now()) {
				if isOnGoing(jb.Status) {
					err := errors.Errorf("reject job submitting: job %s with ID %d is %s", jb.Name, jb.ID, jb.Status)
					aj.SendInternalServerError(errors.Wrap(err, "submit : AJAPI"))
					return
				}

				// For scan all job, check more
				if jb.Name == common_job.ImageScanAllJob {
					// Get the overall stats with the ID of the previous job
					stats, err := scan.DefaultController.GetStats(fmt.Sprintf("%d", jb.ID))
					if err != nil {
						aj.SendInternalServerError(errors.Wrap(err, "submit : AJAPI"))
						return
					}

					if stats.Total != stats.Completed {
						// Not all scan processes are completed
						err := errors.Errorf("scan processes started by %s job with ID %d is in progress: %s", jb.Name, jb.ID, progress(stats.Completed, stats.Total))
						aj.SendPreconditionFailedError(errors.Wrap(err, "submit : AJAPI"))
						return
					}
				}
			}
		}
	}

	id, err := dao.AddAdminJob(&common_models.AdminJob{
		Name:       ajr.Name,
		Kind:       ajr.JobKind(),
		Cron:       ajr.CronString(),
		Parameters: ajr.ParamString(),
	})
	if err != nil {
		aj.SendInternalServerError(err)
		return
	}
	ajr.ID = id
	job := ajr.ToJob()

	// submit job to job service
	log.Debugf("submitting admin job to job service")
	uuid, err := utils_core.GetJobServiceClient().SubmitJob(job)
	if err != nil {
		if err := dao.DeleteAdminJob(id); err != nil {
			log.Debugf("Failed to delete admin job, err: %v", err)
		}
		aj.ParseAndHandleError("failed to submit admin job", err)
		return
	}
	if err := dao.SetAdminJobUUID(id, uuid); err != nil {
		aj.SendInternalServerError(err)
		return
	}
}

func (aj *AJAPI) getLatestAdminJob(name, kind string) (*common_models.AdminJob, error) {
	query := &common_models.AdminJobQuery{
		Name: name,
		Kind: kind,
	}
	query.Size = 1
	query.Page = 1

	jbs, err := dao.GetAdminJobs(query)

	if err != nil {
		return nil, err
	}

	if len(jbs) == 0 {
		// Not exist
		return nil, nil
	}

	// Return the latest one (with biggest ID)
	return jbs[0], nil
}

func convertToAdminJobRep(job *common_models.AdminJob) (models.AdminJobRep, error) {
	if job == nil {
		return models.AdminJobRep{}, nil
	}

	AdminJobRep := models.AdminJobRep{
		ID:           job.ID,
		Name:         job.Name,
		Kind:         job.Kind,
		Status:       job.Status,
		Parameters:   job.Parameters,
		CreationTime: job.CreationTime,
		UpdateTime:   job.UpdateTime,
	}

	if len(job.Cron) > 0 {
		schedule, err := models.ConvertSchedule(job.Cron)
		if err != nil {
			return models.AdminJobRep{}, err
		}
		AdminJobRep.Schedule = &schedule
	}
	return AdminJobRep, nil
}

func progress(completed, total uint) string {
	if total == 0 {
		return fmt.Sprintf("0%s", "%")
	}

	v := float64(completed)
	vv := float64(total)

	p := (int)(math.Round((v / vv) * 100))

	return fmt.Sprintf("%d%s", p, "%")
}

func isOnGoing(status string) bool {
	return status == common_models.JobRunning ||
		status == common_models.JobScheduled ||
		status == common_models.JobPending
}
<commit_msg>fix gc job submit issue<commit_after>// Copyright 2018 Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"
	"math"
	"net/http"
	"strconv"
	"time"

	"github.com/goharbor/harbor/src/common/dao"
	common_http "github.com/goharbor/harbor/src/common/http"
	common_job "github.com/goharbor/harbor/src/common/job"
	common_models "github.com/goharbor/harbor/src/common/models"
	"github.com/goharbor/harbor/src/controller/scan"
	"github.com/goharbor/harbor/src/core/api/models"
	utils_core "github.com/goharbor/harbor/src/core/utils"
	"github.com/goharbor/harbor/src/lib/errors"
	"github.com/goharbor/harbor/src/lib/log"
)

// AJAPI manages the CRUD of admin job and its schedule, any API wants to handle manual and cron job like ScanAll and GC cloud reuse it.
type AJAPI struct {
	BaseController
}

// Prepare validates the URL and parms, it needs the system admin permission.
func (aj *AJAPI) Prepare() {
	aj.BaseController.Prepare()
}

// updateSchedule update a schedule of admin job.
func (aj *AJAPI) updateSchedule(ajr models.AdminJobReq) {
	if ajr.Schedule.Type == models.ScheduleManual {
		aj.SendInternalServerError((fmt.Errorf("fail to update admin job schedule as wrong schedule type: %s", ajr.Schedule.Type)))
		return
	}

	query := &common_models.AdminJobQuery{
		Name: ajr.Name,
		Kind: common_job.JobKindPeriodic,
	}
	jobs, err := dao.GetAdminJobs(query)
	if err != nil {
		aj.SendInternalServerError(err)
		return
	}
	if len(jobs) != 1 {
		aj.SendInternalServerError(errors.New("fail to update admin job schedule as we found more than one schedule in system, please ensure that only one schedule left for your job"))
		return
	}

	// stop the scheduled job and remove it.
	if err = utils_core.GetJobServiceClient().PostAction(jobs[0].UUID, common_job.JobActionStop); err != nil {
		_, ok := err.(*common_job.StatusBehindError)
		if !ok {
			if e, ok := err.(*common_http.Error); !ok || e.Code != http.StatusNotFound {
				aj.SendInternalServerError(err)
				return
			}
		}
	}

	if err = dao.DeleteAdminJob(jobs[0].ID); err != nil {
		aj.SendInternalServerError(err)
		return
	}

	// Set schedule to None means to cancel the schedule, won't add new job.
	if ajr.Schedule.Type != models.ScheduleNone {
		aj.submit(&ajr)
	}
}

// get get a execution of admin job by ID
func (aj *AJAPI) get(id int64) {
	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
		ID: id,
	})
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("failed to get admin jobs: %v", err))
		return
	}
	if len(jobs) == 0 {
		aj.SendNotFoundError(errors.New("no admin job found"))
		return
	}

	adminJobRep, err := convertToAdminJobRep(jobs[0])
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("failed to convert admin job response: %v", err))
		return
	}

	aj.Data["json"] = adminJobRep
	aj.ServeJSON()
}

// list list all executions of admin job by name
func (aj *AJAPI) list(name string) {
	jobs, err := dao.GetTop10AdminJobsOfName(name)
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("failed to get admin jobs: %v", err))
		return
	}

	AdminJobReps := []*models.AdminJobRep{}
	for _, job := range jobs {
		AdminJobRep, err := convertToAdminJobRep(job)
		if err != nil {
			aj.SendInternalServerError(fmt.Errorf("failed to convert admin job response: %v", err))
			return
		}
		AdminJobReps = append(AdminJobReps, &AdminJobRep)
	}

	aj.Data["json"] = AdminJobReps
	aj.ServeJSON()
}

// getSchedule gets admin job schedule ...
func (aj *AJAPI) getSchedule(name string) {
	result := models.AdminJobRep{}

	jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
		Name: name,
		Kind: common_job.JobKindPeriodic,
	})
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("failed to get admin jobs: %v", err))
		return
	}
	if len(jobs) > 1 {
		aj.SendInternalServerError(errors.New("get more than one scheduled admin job, make sure there has only one"))
		return
	}

	if len(jobs) != 0 {
		adminJobRep, err := convertToAdminJobRep(jobs[0])
		if err != nil {
			aj.SendInternalServerError(fmt.Errorf("failed to convert admin job response: %v", err))
			return
		}
		result.Schedule = adminJobRep.Schedule
		result.Parameters = adminJobRep.Parameters
	}

	aj.Data["json"] = result
	aj.ServeJSON()
}

// getLog ...
func (aj *AJAPI) getLog(id int64) {
	job, err := dao.GetAdminJob(id)
	if err != nil {
		log.Errorf("Failed to load job data for job: %d, error: %v", id, err)
		aj.SendInternalServerError(errors.New("Failed to get Job data"))
		return
	}
	if job == nil {
		log.Errorf("Failed to get admin job: %d", id)
		aj.SendNotFoundError(errors.New("Failed to get Job"))
		return
	}

	var jobID string
	// to get the latest execution job id, then to query job log.
	if job.Kind == common_job.JobKindPeriodic {
		exes, err := utils_core.GetJobServiceClient().GetExecutions(job.UUID)
		if err != nil {
			aj.SendInternalServerError(err)
			return
		}
		if len(exes) == 0 {
			aj.SendNotFoundError(errors.New("no execution log found"))
			return
		}
		// get the latest terminal status execution.
		for _, exe := range exes {
			if exe.Info.Status == "Error" || exe.Info.Status == "Success" {
				jobID = exe.Info.JobID
				break
			}
		}
		// no execution found
		if jobID == "" {
			aj.SendNotFoundError(errors.New("no execution log found"))
			return
		}

	} else {
		jobID = job.UUID
	}

	logBytes, err := utils_core.GetJobServiceClient().GetJobLog(jobID)
	if err != nil {
		if httpErr, ok := err.(*common_http.Error); ok {
			aj.RenderError(httpErr.Code, "")
			log.Errorf(fmt.Sprintf("failed to get log of job %d: %d %s",
				id, httpErr.Code, httpErr.Message))
			return
		}
		aj.SendInternalServerError(fmt.Errorf("Failed to get job logs, uuid: %s, error: %v", job.UUID, err))
		return
	}
	aj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Length"), strconv.Itoa(len(logBytes)))
	aj.Ctx.ResponseWriter.Header().Set(http.CanonicalHeaderKey("Content-Type"), "text/plain")
	_, err = aj.Ctx.ResponseWriter.Write(logBytes)
	if err != nil {
		aj.SendInternalServerError(fmt.Errorf("Failed to write job logs, uuid: %s, error: %v", job.UUID, err))
	}
}

// submit submits a job to job service per request
func (aj *AJAPI) submit(ajr *models.AdminJobReq) {
	// when the schedule is saved as None without any schedule, just return 200 and do nothing.
	if ajr.Schedule == nil || ajr.Schedule.Type == models.ScheduleNone {
		return
	}

	// cannot post multiple schedule for admin job.
	if ajr.IsPeriodic() {
		jobs, err := dao.GetAdminJobs(&common_models.AdminJobQuery{
			Name: ajr.Name,
			Kind: common_job.JobKindPeriodic,
		})
		if err != nil {
			aj.SendInternalServerError(fmt.Errorf("failed to get admin jobs: %v", err))
			return
		}
		if len(jobs) != 0 {
			aj.SendPreconditionFailedError(errors.New("fail to set schedule for admin job as always had one, please delete it firstly then to re-schedule"))
			return
		}
	} else {
		// So far, it should be a generic job for the manually trigger case.
		// Only needs to care the 1st generic job.
		// Check if there are still ongoing scan jobs triggered by the previous admin job.
		// TODO: REPLACE WITH TASK MANAGER METHODS IN FUTURE
		jb, err := aj.getLatestAdminJob(ajr.Name, common_job.JobKindGeneric)
		if err != nil {
			aj.SendInternalServerError(errors.Wrap(err, "AJAPI"))
			return
		}

		if jb != nil {
			// With a reasonable timeout duration
			if jb.UpdateTime.Add(2 * time.Hour).After(time.Now()) {
				if isOnGoing(jb.Status) {
					err := errors.Errorf("reject job submitting: job %s with ID %d is %s", jb.Name, jb.ID, jb.Status)
					aj.SendConflictError(errors.Wrap(err, "submit : AJAPI"))
					return
				}

				// For scan all job, check more
				if jb.Name == common_job.ImageScanAllJob {
					// Get the overall stats with the ID of the previous job
					stats, err := scan.DefaultController.GetStats(fmt.Sprintf("%d", jb.ID))
					if err != nil {
						aj.SendInternalServerError(errors.Wrap(err, "submit : AJAPI"))
						return
					}

					if stats.Total != stats.Completed {
						// Not all scan processes are completed
						err := errors.Errorf("scan processes started by %s job with ID %d is in progress: %s", jb.Name, jb.ID, progress(stats.Completed, stats.Total))
						aj.SendPreconditionFailedError(errors.Wrap(err, "submit : AJAPI"))
						return
					}
				}
			}
		}
	}

	id, err := dao.AddAdminJob(&common_models.AdminJob{
		Name:       ajr.Name,
		Kind:       ajr.JobKind(),
		Cron:       ajr.CronString(),
		Parameters: ajr.ParamString(),
	})
	if err != nil {
		aj.SendInternalServerError(err)
		return
	}
	ajr.ID = id
	job := ajr.ToJob()

	// submit job to job service
	log.Debugf("submitting admin job to job service")
	uuid, err := utils_core.GetJobServiceClient().SubmitJob(job)
	if err != nil {
		if err := dao.DeleteAdminJob(id); err != nil {
			log.Debugf("Failed to delete admin job, err: %v", err)
		}
		aj.ParseAndHandleError("failed to submit admin job", err)
		return
	}
	if err := dao.SetAdminJobUUID(id, uuid); err != nil {
		aj.SendInternalServerError(err)
		return
	}
}

func (aj *AJAPI) getLatestAdminJob(name, kind string) (*common_models.AdminJob, error) {
	query := &common_models.AdminJobQuery{
		Name: name,
		Kind: kind,
	}
	query.Size = 1
	query.Page = 1

	jbs, err := dao.GetAdminJobs(query)

	if err != nil {
		return nil, err
	}

	if len(jbs) == 0 {
		// Not exist
		return nil, nil
	}

	// Return the latest one (with biggest ID)
	return jbs[0], nil
}

func convertToAdminJobRep(job *common_models.AdminJob) (models.AdminJobRep, error) {
	if job == nil {
		return models.AdminJobRep{}, nil
	}

	AdminJobRep := models.AdminJobRep{
		ID:           job.ID,
		Name:         job.Name,
		Kind:         job.Kind,
		Status:       job.Status,
		Parameters:   job.Parameters,
		CreationTime: job.CreationTime,
		UpdateTime:   job.UpdateTime,
	}

	if len(job.Cron) > 0 {
		schedule, err := models.ConvertSchedule(job.Cron)
		if err != nil {
			return models.AdminJobRep{}, err
		}
		AdminJobRep.Schedule = &schedule
	}
	return AdminJobRep, nil
}

func progress(completed, total uint) string {
	if total == 0 {
		return fmt.Sprintf("0%s", "%")
	}

	v := float64(completed)
	vv := float64(total)

	p := (int)(math.Round((v / vv) * 100))

	return fmt.Sprintf("%d%s", p, "%")
}

func isOnGoing(status string) bool {
	return status == common_models.JobRunning ||
		status == common_models.JobScheduled ||
		status == common_models.JobPending
}
<|endoftext|>