<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"regexp"
	"runtime"

	"github.com/pkg/errors"
)

type jsonObjectRaw map[string]json.RawMessage
type jsonObject map[string]interface{}

type tilePosition struct {
	collection string
	path       string
}

// Log to standard error
var stderr = log.New(os.Stderr, "", 0)

// List of known file position
var fileList = make(map[string][]tilePosition, 4096)

var nameSelect = regexp.MustCompile(`[^:/\\]+$`)
var tilepathCut = regexp.MustCompile(`:/{0,2}`)

func getJSONRawSlice(r jsonObjectRaw, k string) (*[]jsonObjectRaw, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var rawSlice []jsonObjectRaw
	err := json.Unmarshal(blob, &rawSlice)
	if err != nil {
		return nil, errors.Wrap(err, "JSON unmarshal failed")
	}
	return &rawSlice, nil
}

func getJSONSlice(r jsonObjectRaw, k string) (*[]jsonObject, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedSlice []jsonObject
	err := json.Unmarshal(blob, &decodedSlice)
	if err != nil {
		return nil, errors.Wrap(err, "JSON unmarshal failed")
	}
	return &decodedSlice, nil
}

func getJSONRawObject(r jsonObjectRaw, k string) (*jsonObjectRaw, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedObject jsonObjectRaw
	err := json.Unmarshal(blob, &decodedObject)
	if err != nil {
		return nil, errors.Wrap(err, "JSON unmarshal failed")
	}
	return &decodedObject, nil
}

func getJSONObject(r jsonObjectRaw, k string) (*jsonObject, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedObject jsonObject
	err := json.Unmarshal(blob, &decodedObject)
	if err != nil {
		return nil, errors.Wrap(err, "JSON unmarshal failed")
	}
	return &decodedObject, nil
}

func readMapFile(path string) (*jsonObjectRaw, error) {
	// Read the file
	mapBlob, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, errors.Wrap(err, "read failed")
	}
	// Decode it in hexMap
	var hexMap jsonObjectRaw
	err = json.Unmarshal(mapBlob, &hexMap)
	if err != nil {
		return nil, errors.Wrap(err, "map file decode failed")
	}
	return &hexMap, nil
}

func findHomeDir() string {
	u, err := user.Current()
	if err != nil {
		stderr.Fatal("Error: unable to get the username: ", err)
	}
	return u.HomeDir
}

func readSettingsBlob(userConfig string) ([]byte, error) {
	settings := filepath.Join(userConfig, "hex-kit", "Settings")
	stderr.Print("Reading user settings from: ", settings)
	settingsBlob, err := ioutil.ReadFile(settings)
	return settingsBlob, err
}

// https://electron.atom.io/docs/api/app/#appgetpathname
func getSettings() (jsonObjectRaw, error) {
	var userConfig string
	var settingsBlob []byte
	var settingsRaw jsonObjectRaw
	var err error
	switch runtime.GOOS {
	case "darwin":
		homeDir := findHomeDir()
		userConfig = filepath.Join(homeDir, "Library", "Application Support")
		settingsBlob, err = readSettingsBlob(userConfig)
	case "linux":
		userConfig = os.Getenv("XDG_CONFIG_HOME")
		if userConfig != "" {
			settingsBlob, err = readSettingsBlob(userConfig)
		}
		if userConfig == "" || err != nil {
			homeDir := findHomeDir()
			userConfig = filepath.Join(homeDir, ".config")
			settingsBlob, err = readSettingsBlob(userConfig)
		}
	case "windows":
		userConfig = os.Getenv("APPDATA")
		if userConfig == "" {
			return nil, errors.New("Error: unable to find user config (no APPDATA environment variable)")
		}
		settingsBlob, err = readSettingsBlob(userConfig)
	}
	// Did the log read succeed
	if err != nil {
		return nil, errors.Wrap(err, "unable to read settings")
	}
	err = json.Unmarshal(settingsBlob, &settingsRaw)
	if err != nil {
		return nil, errors.Wrap(err, "unable to decode settings")
	}
	return settingsRaw, nil
}

// Search fon all png files under the current path
func pathMap(collectionName, basePath string) filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			stderr.Println("Warning: while searching for PNG files: ", err)
			return nil
		}
		tileName := info.Name()
		lenPath := len(path)
		if lenPath > 4 && path[lenPath-4:] == ".png" {
			relPathTile, _ := filepath.Rel(basePath, path)
			var tp tilePosition
			tp.collection = collectionName
			tp.path = filepath.ToSlash(filepath.Clean(relPathTile))
			fileList[tileName] = append(fileList[tileName], tp)
		}
		return nil
	}
}

func getCollectionDir(settings jsonObjectRaw) (*map[string]string, error) {
	// Build the list of collections
	collectionsDir := make(map[string]string)
	collections, err := getJSONRawObject(settings, "tiles")
	if err != nil {
		return nil, errors.Wrap(err, "unable to access the \"tiles\" list")
	}
	for name, collectionBlob := range *collections {
		var collection jsonObject
		err := json.Unmarshal(collectionBlob, &collection)
		if err != nil {
			return nil, errors.Wrapf(err, "unable to parse tiles[%s]", name)
		}
		// Ignore source if hidden
		hiddenIntf, ok := collection["hidden"]
		if ok {
			hidden, isBool := hiddenIntf.(bool)
			if isBool && hidden {
				continue
			}
		}
		pathIntf, ok := collection["path"]
		if !ok {
			return nil, errors.Wrapf(err, "no path for %s", name)
		}
		path, ok := pathIntf.(string)
		if !ok {
			return nil, errors.Wrapf(err, "the path for %s is not a string", name)
		}
		// Relative collection path
		if !filepath.IsAbs(path) {
			collectionsDir[name] = filepath.Join(os.Args[1], "resources", "app.asar.unpacked", path)
			continue
		}
		// Absolute collection path
		collectionsDir[name] = path
	}
	return &collectionsDir, nil

}

func main() {
	if len(os.Args) != 3 {
		stderr.Println("Usage:", os.Args[0], "HexkitPath MapPath")
		return
	}
	// Read the settngs and get the
	settings, err := getSettings()
	if err != nil {
		log.Fatal("Unable to read user settings: ", err)
	}
	collectionsDir, err := getCollectionDir(settings)
	if err != nil {
		log.Fatal("Unable to read the list of collections: ", err)
	}
	// Build the list of PNG files
	for name, path := range *collectionsDir {
		err := filepath.Walk(path, pathMap(name, path))
		if err != nil {
			stderr.Fatal(err)
		}
	}
	// Read the file
	hexMap, err := readMapFile(os.Args[2])
	if err != nil {
		stderr.Fatal("Error: unable to read map file: ", err)
	}
	// Get the layers list
	layers, err := getJSONRawSlice(*hexMap, "layers")
	if err != nil {
		stderr.Fatal("Map format error", err)
	}

	// Search for tiles
	layersModified := false
	for i, v := range *layers {
		tiles, err := getJSONSlice(v, "tiles")
		if err != nil {
			stderr.Fatal("Map format error in layer ", i+1, ": ", err)
		}
		// Search for the source of tiles
		tilesModified := false
		for j, t := range *tiles {
			// Ignore undefined tiles
			if t == nil {
				continue
			}
			sourceBlob, ok := t["source"]
			if !ok {
				stderr.Println("Warning: layer", i+1, "tile", j+1, "no tile source found")
				continue
			}
			source, ok := sourceBlob.(string)
			if !ok {
				stderr.Println("Warning: layer", i+1, "tile", j+1, "the tile source is incorrect (not a string)")
				continue
			}
			// Skip the default blank tiles
			if source[:6] == "Blank:" {
				continue
			}
			// Have we found the tile
			fileName := nameSelect.FindString(source)
			pathList, ok := fileList[fileName]
			if !ok {
				stderr.Println("Warning: layer", i+1, "tile", j+1, "unable to find tile image file for", source)
				continue
			}
			// Search for the current path in the
			firstSplit := tilepathCut.Split(source, 2)
			if len(firstSplit) < 2 {
				stderr.Println("Warning: layer", i+1, "tile", j+1, "incorrect source: ", source)
				continue
			}
			targetCollection := firstSplit[0]
			targetPath := firstSplit[1]
			var bestScore int
			var selected tilePosition
		pathSearch:
			for _, p := range pathList {
				if targetCollection == p.collection && targetPath == p.path {
					break pathSearch
				}
				currentScore := len(p.path)
				if targetCollection == p.collection && (currentScore+256) > bestScore {
					bestScore = currentScore + 256
					selected = p
				}
				if currentScore > bestScore {
					bestScore = currentScore
					selected = p
				}
			}
			// A new value was found: update the source
			if selected.collection != "" && selected.path != "" {
				tilesModified = true
				t["source"] = selected.collection + "://" + selected.path
			}
		}
		if tilesModified {
			tilesBlob, err := json.Marshal(tiles)
			if err != nil {
				stderr.Fatal(err)
			}
			v["tiles"] = tilesBlob
			layersModified = true
		}
	}
	if layersModified {
		layersBlob, err := json.Marshal(layers)
		if err != nil {
			stderr.Fatal(err)
		}
		(*hexMap)["layers"] = layersBlob
	}
	b, err := json.Marshal(*hexMap)
	if err != nil {
		stderr.Fatal(err)
	}
	_, err = os.Stdout.Write(b)
	if err != nil {
		stderr.Fatal(err)
	}
}
<commit_msg>Update findHomeDir to return an error value<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/user"
	"path/filepath"
	"regexp"
	"runtime"

	"github.com/pkg/errors"
)

type jsonObjectRaw map[string]json.RawMessage
type jsonObject map[string]interface{}

type tilePosition struct {
	collection string
	path       string
}

// Log to standard error
var stderr = log.New(os.Stderr, "", 0)

// List of known file position
var fileList = make(map[string][]tilePosition, 4096)

var nameSelect = regexp.MustCompile(`[^:/\\]+$`)
var tilepathCut = regexp.MustCompile(`:/{0,2}`)

func getJSONRawSlice(r jsonObjectRaw, k string) (*[]jsonObjectRaw, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var rawSlice []jsonObjectRaw
	err := json.Unmarshal(blob, &rawSlice)
	if err != nil {
		return nil, errors.Wrap(err, "JSON unmarshal failed")
	}
	return &rawSlice, nil
}

func getJSONSlice(r jsonObjectRaw, k string) (*[]jsonObject, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedSlice []jsonObject
	err := json.Unmarshal(blob, &decodedSlice)
	if err != nil {
		return nil, errors.Wrap(err, "JSON unmarshal failed")
	}
	return &decodedSlice, nil
}

func getJSONRawObject(r jsonObjectRaw, k string) (*jsonObjectRaw, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedObject jsonObjectRaw
	err := json.Unmarshal(blob, &decodedObject)
	if err != nil {
		return nil, errors.Wrap(err, "JSON unmarshal failed")
	}
	return &decodedObject, nil
}

func getJSONObject(r jsonObjectRaw, k string) (*jsonObject, error) {
	blob, ok := r[k]
	if !ok {
		return nil, fmt.Errorf("no \"%s\" key in object", k)
	}
	var decodedObject jsonObject
	err := json.Unmarshal(blob, &decodedObject)
	if err != nil {
		return nil, errors.Wrap(err, "JSON unmarshal failed")
	}
	return &decodedObject, nil
}

func readMapFile(path string) (*jsonObjectRaw, error) {
	// Read the file
	mapBlob, err := ioutil.ReadFile(path)
	if err != nil {
		return nil, errors.Wrap(err, "read failed")
	}
	// Decode it in hexMap
	var hexMap jsonObjectRaw
	err = json.Unmarshal(mapBlob, &hexMap)
	if err != nil {
		return nil, errors.Wrap(err, "map file decode failed")
	}
	return &hexMap, nil
}

func findHomeDir() (string, error) {
	u, err := user.Current()
	if err != nil {
		return "", errors.Wrap(err, "unable to get user information: ")
	}
	return u.HomeDir, nil
}

func readSettingsBlob(userConfig string) ([]byte, error) {
	settings := filepath.Join(userConfig, "hex-kit", "Settings")
	stderr.Print("Reading user settings from: ", settings)
	settingsBlob, err := ioutil.ReadFile(settings)
	return settingsBlob, err
}

// https://electron.atom.io/docs/api/app/#appgetpathname
func getSettings() (jsonObjectRaw, error) {
	var userConfig string
	var settingsBlob []byte
	var settingsRaw jsonObjectRaw
	var err error
	switch runtime.GOOS {
	case "darwin":
		homeDir, err := findHomeDir()
		if err != nil {
			return nil, err
		}
		userConfig = filepath.Join(homeDir, "Library", "Application Support")
		settingsBlob, err = readSettingsBlob(userConfig)
	case "linux":
		userConfig = os.Getenv("XDG_CONFIG_HOME")
		if userConfig != "" {
			settingsBlob, err = readSettingsBlob(userConfig)
		}
		if userConfig == "" || err != nil {
			homeDir, err := findHomeDir()
			if err != nil {
				return nil, err
			}
			userConfig = filepath.Join(homeDir, ".config")
			settingsBlob, err = readSettingsBlob(userConfig)
		}
	case "windows":
		userConfig = os.Getenv("APPDATA")
		if userConfig == "" {
			return nil, errors.New("Error: unable to find user config (no APPDATA environment variable)")
		}
		settingsBlob, err = readSettingsBlob(userConfig)
	}
	// Did the log read succeed
	if err != nil {
		return nil, errors.Wrap(err, "unable to read settings")
	}
	err = json.Unmarshal(settingsBlob, &settingsRaw)
	if err != nil {
		return nil, errors.Wrap(err, "unable to decode settings")
	}
	return settingsRaw, nil
}

// Search fon all png files under the current path
func pathMap(collectionName, basePath string) filepath.WalkFunc {
	return func(path string, info os.FileInfo, err error) error {
		if err != nil {
			stderr.Println("Warning: while searching for PNG files: ", err)
			return nil
		}
		tileName := info.Name()
		lenPath := len(path)
		if lenPath > 4 && path[lenPath-4:] == ".png" {
			relPathTile, _ := filepath.Rel(basePath, path)
			var tp tilePosition
			tp.collection = collectionName
			tp.path = filepath.ToSlash(filepath.Clean(relPathTile))
			fileList[tileName] = append(fileList[tileName], tp)
		}
		return nil
	}
}

func getCollectionDir(settings jsonObjectRaw) (*map[string]string, error) {
	// Build the list of collections
	collectionsDir := make(map[string]string)
	collections, err := getJSONRawObject(settings, "tiles")
	if err != nil {
		return nil, errors.Wrap(err, "unable to access the \"tiles\" list")
	}
	for name, collectionBlob := range *collections {
		var collection jsonObject
		err := json.Unmarshal(collectionBlob, &collection)
		if err != nil {
			return nil, errors.Wrapf(err, "unable to parse tiles[%s]", name)
		}
		// Ignore source if hidden
		hiddenIntf, ok := collection["hidden"]
		if ok {
			hidden, isBool := hiddenIntf.(bool)
			if isBool && hidden {
				continue
			}
		}
		pathIntf, ok := collection["path"]
		if !ok {
			return nil, errors.Wrapf(err, "no path for %s", name)
		}
		path, ok := pathIntf.(string)
		if !ok {
			return nil, errors.Wrapf(err, "the path for %s is not a string", name)
		}
		// Relative collection path
		if !filepath.IsAbs(path) {
			collectionsDir[name] = filepath.Join(os.Args[1], "resources", "app.asar.unpacked", path)
			continue
		}
		// Absolute collection path
		collectionsDir[name] = path
	}
	return &collectionsDir, nil

}

func main() {
	if len(os.Args) != 3 {
		stderr.Println("Usage:", os.Args[0], "HexkitPath MapPath")
		return
	}
	// Read the settngs and get the
	settings, err := getSettings()
	if err != nil {
		log.Fatal("Unable to read user settings: ", err)
	}
	collectionsDir, err := getCollectionDir(settings)
	if err != nil {
		log.Fatal("Unable to read the list of collections: ", err)
	}
	// Build the list of PNG files
	for name, path := range *collectionsDir {
		err := filepath.Walk(path, pathMap(name, path))
		if err != nil {
			stderr.Fatal(err)
		}
	}
	// Read the file
	hexMap, err := readMapFile(os.Args[2])
	if err != nil {
		stderr.Fatal("Error: unable to read map file: ", err)
	}
	// Get the layers list
	layers, err := getJSONRawSlice(*hexMap, "layers")
	if err != nil {
		stderr.Fatal("Map format error", err)
	}

	// Search for tiles
	layersModified := false
	for i, v := range *layers {
		tiles, err := getJSONSlice(v, "tiles")
		if err != nil {
			stderr.Fatal("Map format error in layer ", i+1, ": ", err)
		}
		// Search for the source of tiles
		tilesModified := false
		for j, t := range *tiles {
			// Ignore undefined tiles
			if t == nil {
				continue
			}
			sourceBlob, ok := t["source"]
			if !ok {
				stderr.Println("Warning: layer", i+1, "tile", j+1, "no tile source found")
				continue
			}
			source, ok := sourceBlob.(string)
			if !ok {
				stderr.Println("Warning: layer", i+1, "tile", j+1, "the tile source is incorrect (not a string)")
				continue
			}
			// Skip the default blank tiles
			if source[:6] == "Blank:" {
				continue
			}
			// Have we found the tile
			fileName := nameSelect.FindString(source)
			pathList, ok := fileList[fileName]
			if !ok {
				stderr.Println("Warning: layer", i+1, "tile", j+1, "unable to find tile image file for", source)
				continue
			}
			// Search for the current path in the
			firstSplit := tilepathCut.Split(source, 2)
			if len(firstSplit) < 2 {
				stderr.Println("Warning: layer", i+1, "tile", j+1, "incorrect source: ", source)
				continue
			}
			targetCollection := firstSplit[0]
			targetPath := firstSplit[1]
			var bestScore int
			var selected tilePosition
		pathSearch:
			for _, p := range pathList {
				if targetCollection == p.collection && targetPath == p.path {
					break pathSearch
				}
				currentScore := len(p.path)
				if targetCollection == p.collection && (currentScore+256) > bestScore {
					bestScore = currentScore + 256
					selected = p
				}
				if currentScore > bestScore {
					bestScore = currentScore
					selected = p
				}
			}
			// A new value was found: update the source
			if selected.collection != "" && selected.path != "" {
				tilesModified = true
				t["source"] = selected.collection + "://" + selected.path
			}
		}
		if tilesModified {
			tilesBlob, err := json.Marshal(tiles)
			if err != nil {
				stderr.Fatal(err)
			}
			v["tiles"] = tilesBlob
			layersModified = true
		}
	}
	if layersModified {
		layersBlob, err := json.Marshal(layers)
		if err != nil {
			stderr.Fatal(err)
		}
		(*hexMap)["layers"] = layersBlob
	}
	b, err := json.Marshal(*hexMap)
	if err != nil {
		stderr.Fatal(err)
	}
	_, err = os.Stdout.Write(b)
	if err != nil {
		stderr.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package http

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"time"

	gostLog "github.com/gost/server/log"
	"github.com/gost/server/sensorthings/models"
	log "github.com/sirupsen/logrus"
)

var logger *log.Entry

func setupLogger() {
	l, err := gostLog.GetLoggerInstance()
	if err != nil {
		log.Error(err)
	}

	//Setting default fields for main logger
	logger = l.WithFields(log.Fields{"package": "gost.server.http"})
}

// Server interface for starting and stopping the HTTP server
type Server interface {
	Start()
	Stop()
}

// GostServer is the type that contains all of the relevant information to set
// up the GOST HTTP Server
type GostServer struct {
	host       string      // Hostname for example "localhost" or "192.168.1.14"
	port       int         // Port number where you want to run your http server on
	api        *models.API // SensorThings api to interact with from the HttpServer
	https      bool
	httpsCert  string
	httpsKey   string
	httpServer *http.Server
}

// CreateServer initialises a new GOST HTTPServer based on the given parameters
func CreateServer(host string, port int, api *models.API, https bool, httpsCert, httpsKey string) Server {
	setupLogger()
	a := *api
	router := CreateRouter(api)
	return &GostServer{
		host:      host,
		port:      port,
		api:       api,
		https:     https,
		httpsCert: httpsCert,
		httpsKey:  httpsKey,
		httpServer: &http.Server{
			Addr:         fmt.Sprintf("%s:%s", host, strconv.Itoa(port)),
			Handler:      PostProcessHandler(LowerCaseURI(router), a.GetConfig().Server.ExternalURI),
			ReadTimeout:  30 * time.Second,
			WriteTimeout: 30 * time.Second,
		},
	}
}

// Start command to start the GOST HTTPServer
func (s *GostServer) Start() {
	t := "HTTP"
	if s.https {
		t = "HTTPS"
	}

	logger.Infof("Started GOST %v Server on %v:%v", t, s.host, s.port)

	var err error
	if s.https {
		err = s.httpServer.ListenAndServeTLS(s.httpsCert, s.httpsKey)
	} else {
		err = s.httpServer.ListenAndServe()
	}

	if err != nil {
		logger.Panicf("GOST server not properly stopped: %v", err)
	}
}

// Stop command to stop the GOST HTTP server
func (s *GostServer) Stop() {
	if s.httpServer != nil {
		logger.Info("Stopping HTTP(S) Server")
		s.httpServer.Shutdown(context.Background())
	}
}

// LowerCaseURI is a middleware function that lower cases the url path
func LowerCaseURI(h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		lowerCasePath := strings.ToLower(r.URL.Path)
		// temporarily disabled checking on paths due to problems in serving with /$value
		/*
			api := *s.api
			split := strings.Split(lowerCasePath, "/")
			for i, s := range split {
				if len(s) == 0 || i+1 == len(split) {
					continue
				}

				found := false
				for _, a := range api.GetAcceptedPaths() {
					if strings.HasPrefix(s, a) {
						found = true
					}
				}

				if !found {
					w.Header().Set("Content-Type", "application/json; charset=UTF-8")
					w.WriteHeader(http.StatusNotFound)
					w.Write([]byte(""))
					return
				}
			}
		*/

		r.URL.RawPath = r.URL.Path
		r.URL.Path = lowerCasePath
		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

// PostProcessHandler runs after all the other handlers and can be used to modify
// the response. In this case modify links (due to proxy running) or handle CORS functionality
// Basically we catch all the response using httptest.NewRecorder (headers + body), modify and
// write to response. Is this a right approach?
func PostProcessHandler(h http.Handler, externalURI string) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		if logger.Logger.Level == log.DebugLevel {
			logger.Debugf("%s start: %s", r.Method, r.URL.Path)
			defer gostLog.DebugfWithElapsedTime(logger, time.Now(), "%s done: %s", r.Method, r.URL.Path)
		}

		origURI := externalURI
		forwardedURI := r.Header.Get("X-Forwarded-For")

		rec := httptest.NewRecorder()

		// first run the next handler and get results
		h.ServeHTTP(rec, r)

		// read response body and replace links
		bytes := rec.Body.Bytes()
		s := string(bytes)
		if len(s) > 0 {
			if len(forwardedURI) > 0 {
				// if both are changed (X-Forwarded-For and External uri environment variabele) use the last one
				if origURI == "http://localhost:8080/" {
					s = strings.Replace(s, origURI, forwardedURI, -1)
				}
			}

		}
		// handle headers too...
		for k, v := range rec.HeaderMap {
			val := v[0]

			// if there is a location header and a proxy running, change
			// the location url too
			if k == "Location" && len(forwardedURI) > 0 {
				if origURI == "http://localhost:8080/" {
					logger.Debugf("proxy + location header detected. forwarded uri: %s", forwardedURI)
					val = strings.Replace(val, origURI, forwardedURI, -1)
				}
			}

			// add the header to response
			w.Header().Add(k, val)
		}

		// handle status code
		w.WriteHeader(rec.Code)

		// write modified response
		w.Write([]byte(s))
	}
	return http.HandlerFunc(fn)
}
<commit_msg>fix external uri in case of proxy<commit_after>package http

import (
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strconv"
	"strings"
	"time"

	gostLog "github.com/gost/server/log"
	"github.com/gost/server/sensorthings/models"
	log "github.com/sirupsen/logrus"
)

var logger *log.Entry

func setupLogger() {
	l, err := gostLog.GetLoggerInstance()
	if err != nil {
		log.Error(err)
	}

	//Setting default fields for main logger
	logger = l.WithFields(log.Fields{"package": "gost.server.http"})
}

// Server interface for starting and stopping the HTTP server
type Server interface {
	Start()
	Stop()
}

// GostServer is the type that contains all of the relevant information to set
// up the GOST HTTP Server
type GostServer struct {
	host       string      // Hostname for example "localhost" or "192.168.1.14"
	port       int         // Port number where you want to run your http server on
	api        *models.API // SensorThings api to interact with from the HttpServer
	https      bool
	httpsCert  string
	httpsKey   string
	httpServer *http.Server
}

// CreateServer initialises a new GOST HTTPServer based on the given parameters
func CreateServer(host string, port int, api *models.API, https bool, httpsCert, httpsKey string) Server {
	setupLogger()
	a := *api
	router := CreateRouter(api)
	return &GostServer{
		host:      host,
		port:      port,
		api:       api,
		https:     https,
		httpsCert: httpsCert,
		httpsKey:  httpsKey,
		httpServer: &http.Server{
			Addr:         fmt.Sprintf("%s:%s", host, strconv.Itoa(port)),
			Handler:      PostProcessHandler(LowerCaseURI(router), a.GetConfig().Server.ExternalURI),
			ReadTimeout:  30 * time.Second,
			WriteTimeout: 30 * time.Second,
		},
	}
}

// Start command to start the GOST HTTPServer
func (s *GostServer) Start() {
	t := "HTTP"
	if s.https {
		t = "HTTPS"
	}

	logger.Infof("Started GOST %v Server on %v:%v", t, s.host, s.port)

	var err error
	if s.https {
		err = s.httpServer.ListenAndServeTLS(s.httpsCert, s.httpsKey)
	} else {
		err = s.httpServer.ListenAndServe()
	}

	if err != nil {
		logger.Panicf("GOST server not properly stopped: %v", err)
	}
}

// Stop command to stop the GOST HTTP server
func (s *GostServer) Stop() {
	if s.httpServer != nil {
		logger.Info("Stopping HTTP(S) Server")
		s.httpServer.Shutdown(context.Background())
	}
}

// LowerCaseURI is a middleware function that lower cases the url path
func LowerCaseURI(h http.Handler) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {
		lowerCasePath := strings.ToLower(r.URL.Path)
		// temporarily disabled checking on paths due to problems in serving with /$value
		/*
			api := *s.api
			split := strings.Split(lowerCasePath, "/")
			for i, s := range split {
				if len(s) == 0 || i+1 == len(split) {
					continue
				}

				found := false
				for _, a := range api.GetAcceptedPaths() {
					if strings.HasPrefix(s, a) {
						found = true
					}
				}

				if !found {
					w.Header().Set("Content-Type", "application/json; charset=UTF-8")
					w.WriteHeader(http.StatusNotFound)
					w.Write([]byte(""))
					return
				}
			}
		*/

		r.URL.RawPath = r.URL.Path
		r.URL.Path = lowerCasePath
		h.ServeHTTP(w, r)
	}

	return http.HandlerFunc(fn)
}

// PostProcessHandler runs after all the other handlers and can be used to modify
// the response. In this case modify links (due to proxy running) or handle CORS functionality
// Basically we catch all the response using httptest.NewRecorder (headers + body), modify and
// write to response. Is this a right approach?
func PostProcessHandler(h http.Handler, externalURI string) http.Handler {
	fn := func(w http.ResponseWriter, r *http.Request) {

		if logger.Logger.Level == log.DebugLevel {
			logger.Debugf("%s start: %s", r.Method, r.URL.Path)
			defer gostLog.DebugfWithElapsedTime(logger, time.Now(), "%s done: %s", r.Method, r.URL.Path)
		}

		origURI := externalURI
		forwardedURI := r.Header.Get("X-Forwarded-For")

		rec := httptest.NewRecorder()

		// first run the next handler and get results
		h.ServeHTTP(rec, r)

		// read response body and replace links
		bytes := rec.Body.Bytes()
		s := string(bytes)
		if len(s) > 0 {
			if len(forwardedURI) > 0 {
				// if both are changed (X-Forwarded-For and External uri environment variabele) use the last one
				if origURI == "http://localhost:8080/" {
					s = strings.Replace(s, origURI, forwardedURI, -1)
				}
			}

		}
		// handle headers too...
		for k, v := range rec.HeaderMap {
			val := v[0]

			// if there is a location header and a proxy running, change
			// the location url too
			if k == "Location" && len(forwardedURI) > 0 {
				if origURI == "http://localhost:8080/" {
					val = strings.Replace(val, "localhost", forwardedURI, -1)
				}
			}

			// add the header to response
			w.Header().Add(k, val)
		}

		// handle status code
		w.WriteHeader(rec.Code)

		// write modified response
		w.Write([]byte(s))
	}
	return http.HandlerFunc(fn)
}
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	"github.com/ProductHealth/gose4"
	"github.com/golang/glog"
	"io"
	"net/http"
	"net/url"
	"strings"
	"github.com/ProductHealth/gommons/time"
)

type check struct {
	url             *url.URL
	method          string
	statusCode      int
	requestBody     *string
	requestHeaders  *map[string]string
	expectedHeaders *map[string]string
	expectedBody    *string
	config          gose4.Configuration
	requestFunc     func(*http.Request) (*http.Response, error)
}

func New(address, method string, statusCode int, config gose4.Configuration) (*check, error) {
	url, err := url.Parse(address)
	if err != nil {
		return nil, err
	}

	requestFunc := func(req *http.Request) (*http.Response, error) {
		return http.DefaultClient.Do(req)
	}

	return &check{url: url, method: method, statusCode: statusCode, config: config, requestFunc: requestFunc}, nil
}

func (hc *check) RequestBody(content string) {
	hc.requestBody = &content
}

func (hc *check) RequestHeaders(rh map[string]string) {
	hc.requestHeaders = &rh
}

func (hc *check) ExpectedHeaders(eh map[string]string) {
	hc.expectedHeaders = &eh
}

func (hc *check) ExpectedBody(content string) {
	hc.requestBody = &content
}

func (hc *check) Run() gose4.HealthCheckResult {
	sw := time.NewStopWatch()
	var bodyReader io.Reader = nil
	if hc.requestBody != nil {
		bodyReader = strings.NewReader(*(hc.requestBody))
	}
	request, err := http.NewRequest(hc.method, hc.url.String(), bodyReader)
	if err != nil {
		return gose4.Failed(sw.GetDuration(), err.Error())
	}
	response, err := hc.requestFunc(request)
	defer response.Body.Close()
	glog.V(5).Infof("RESPONSE %#v ERR: %s", response, err)
	if err != nil {
		return gose4.Failed(sw.GetDuration(), fmt.Sprintf("Error while requesting %#v: %s", hc.url, err))
	}
	if hc.statusCode != response.StatusCode {
		return gose4.Failed(sw.GetDuration(), fmt.Sprintf("Returned response code %#v does not match required %#v ", response.StatusCode, hc.statusCode))
	}
	return gose4.Ok(sw.GetDuration())
}

func (hc *check) Configuration() gose4.Configuration {
	return hc.config
}
<commit_msg>Moved body.close call<commit_after>package http

import (
	"fmt"
	"github.com/ProductHealth/gose4"
	"github.com/golang/glog"
	"io"
	"net/http"
	"net/url"
	"strings"
	"github.com/ProductHealth/gommons/time"
)

type check struct {
	url             *url.URL
	method          string
	statusCode      int
	requestBody     *string
	requestHeaders  *map[string]string
	expectedHeaders *map[string]string
	expectedBody    *string
	config          gose4.Configuration
	requestFunc     func(*http.Request) (*http.Response, error)
}

func New(address, method string, statusCode int, config gose4.Configuration) (*check, error) {
	url, err := url.Parse(address)
	if err != nil {
		return nil, err
	}

	requestFunc := func(req *http.Request) (*http.Response, error) {
		return http.DefaultClient.Do(req)
	}

	return &check{url: url, method: method, statusCode: statusCode, config: config, requestFunc: requestFunc}, nil
}

func (hc *check) RequestBody(content string) {
	hc.requestBody = &content
}

func (hc *check) RequestHeaders(rh map[string]string) {
	hc.requestHeaders = &rh
}

func (hc *check) ExpectedHeaders(eh map[string]string) {
	hc.expectedHeaders = &eh
}

func (hc *check) ExpectedBody(content string) {
	hc.requestBody = &content
}

func (hc *check) Run() gose4.HealthCheckResult {
	sw := time.NewStopWatch()
	var bodyReader io.Reader = nil
	if hc.requestBody != nil {
		bodyReader = strings.NewReader(*(hc.requestBody))
	}
	request, err := http.NewRequest(hc.method, hc.url.String(), bodyReader)
	if err != nil {
		return gose4.Failed(sw.GetDuration(), err.Error())
	}
	response, err := hc.requestFunc(request)
	glog.V(5).Infof("RESPONSE %#v ERR: %s", response, err)
	if err != nil {
		return gose4.Failed(sw.GetDuration(), fmt.Sprintf("Error while requesting %#v: %s", hc.url, err))
	}
	if response.Body != nil {
		defer response.Body.Close()
	}
	if hc.statusCode != response.StatusCode {
		return gose4.Failed(sw.GetDuration(), fmt.Sprintf("Returned response code %#v does not match required %#v ", response.StatusCode, hc.statusCode))
	}
	return gose4.Ok(sw.GetDuration())
}

func (hc *check) Configuration() gose4.Configuration {
	return hc.config
}
<|endoftext|>


<commit_before>package main

import (
	// "encoding/hex"
	"fmt"
	"os/exec"
	"bytes"
	// "time"
	// "math/big"
	// "github.com/PointCoin/btcutil"
	// "github.com/PointCoin/btcwire"
	// "github.com/PointCoin/btcrpcclient"
	// "github.com/PointCoin/btcjson"
	"strconv"
	"strings"
	// "regexp"
	// "math/rand"
	"log"
)

func main() {

	s := "\"Hello\""
	fmt.Println(s)

	o,_ := strconv.Unquote(s)
	fmt.Println(o)

	// command = pointctl getrawtransaction d2011b19dea6e98ec8bf78bd224856e76b6a9c460bbb347e49adb3dcf457e548
	txid := "d2011b19dea6e98ec8bf78bd224856e76b6a9c460bbb347e49adb3dcf457e548"
	cmd := exec.Command("pointctl", "getrawtransaction", txid)

	var out bytes.Buffer
    cmd.Stdout = &out
    err := cmd.Run()
    if err != nil {
    	log.Fatal(err)
    }
    // fmt.Printf("result: %s\n", out)
    fmt.Println(out.String())

	cmd2 := exec.Command("xargs", "pointctl", "decoderawtransaction")
	cmd2.Stdin = strings.NewReader(out.String())
	var out2 bytes.Buffer
    cmd2.Stdout = &out2
    err2 := cmd2.Run()
    if err2 != nil {
    	log.Fatal(err2)
    }

    fmt.Println(out2.String())

}<commit_msg>Reading JSON results<commit_after>package main

import (
	// "encoding/hex"
	"fmt"
	"os/exec"
	"bytes"
	// "time"
	// "math/big"
	// "github.com/PointCoin/btcutil"
	// "github.com/PointCoin/btcwire"
	// "github.com/PointCoin/btcrpcclient"
	// "github.com/PointCoin/btcjson"
	"strconv"
	"strings"
	"encoding/json"
	// "regexp"
	// "math/rand"
	"log"
)

func main() {

	s := "\"Hello\""
	fmt.Println(s)

	o,_ := strconv.Unquote(s)
	fmt.Println(o)

	// command = pointctl getrawtransaction d2011b19dea6e98ec8bf78bd224856e76b6a9c460bbb347e49adb3dcf457e548
	txid := "d2011b19dea6e98ec8bf78bd224856e76b6a9c460bbb347e49adb3dcf457e548"
	cmd := exec.Command("pointctl", "getrawtransaction", txid)

	var out bytes.Buffer
    cmd.Stdout = &out
    err := cmd.Run()
    if err != nil {
    	log.Fatal(err)
    }
    // fmt.Printf("result: %s\n", out)
    fmt.Println(out.String())

	cmd2 := exec.Command("xargs", "pointctl", "decoderawtransaction")
	cmd2.Stdin = strings.NewReader(out.String())
	var out2 bytes.Buffer
    cmd2.Stdout = &out2
    err2 := cmd2.Run()
    if err2 != nil {
    	log.Fatal(err2)
    }

    fmt.Println(out2.String())

	var f interface{}
	err = json.Unmarshal(out2.Bytes(), &f)
	m := f.(map[string]interface{})

	for k, v := range m {
    switch vv := v.(type) {
    case string:
        fmt.Println(k, "is string", vv)
    case int:
        fmt.Println(k, "is int", vv)
    case []interface{}:
        fmt.Println(k, "is an array:")
        for i, u := range vv {
            fmt.Println(i, u)
        }
    default:
        fmt.Println(k, "is of a type I don't know how to handle")
    }
}

}<|endoftext|>


<commit_before>/* Test the httpstress.Test() function. */
package httpstress

import "testing"

func TestTest(t *testing.T) {
	up := []string{"https://google.com", "http://google.com"} // These URLs should pass.
	down := []string{"http://test.invalid"}                   // This should fail.
	invalid := []string{"ftp://localhost"}                    // Error. Non HTTP/HTTPS URL.

	if _, err := Test(1, 1, invalid); err == nil {
		t.Errorf("%s is ok (should be an error)", invalid)
	}

	if err, _ := Test(1, 1, up); len(err) > 0 {
		t.Errorf("%s is down (should be up)", up)
	}

	if err, _ := Test(1, 1, down); len(err) == 0 {
		t.Errorf("%s is up (should be down)", down)
	}
}
<commit_msg>style fix<commit_after>// Test the httpstress.Test() function.
package httpstress

import "testing"

func TestTest(t *testing.T) {
	up := []string{"https://google.com", "http://google.com"} // These URLs should pass.
	down := []string{"http://test.invalid"}                   // This should fail.
	invalid := []string{"ftp://localhost"}                    // Error. Non HTTP/HTTPS URL.

	if _, err := Test(1, 1, invalid); err == nil {
		t.Errorf("%s is ok (should be an error)", invalid)
	}

	if err, _ := Test(1, 1, up); len(err) > 0 {
		t.Errorf("%s is down (should be up)", up)
	}

	if err, _ := Test(1, 1, down); len(err) == 0 {
		t.Errorf("%s is up (should be down)", down)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package networking

const (
	// GroupName is the name for the networking API group.
	GroupName = "networking.internal.knative.dev"

	// IngressClassAnnotationKey is the annotation for the
	// explicit class of Ingress that a particular resource has
	// opted into. For example,
	//
	//    networking.knative.dev/ingress.class: some-network-impl
	//
	// This uses a different domain because unlike the resource, it is
	// user-facing.
	//
	// The parent resource may use its own annotations to choose the
	// annotation value for the Ingress it uses.  Based on such
	// value a different reconciliation logic may be used (for examples,
	// Istio-based Ingress will reconcile into a VirtualService).
	IngressClassAnnotationKey = "networking.knative.dev/ingress.class"

	// IngressLabelKey is the label key attached to underlying network programming
	// resources to indicate which Ingress triggered their creation.
	IngressLabelKey = GroupName + "/ingress"

	// SKSLabelKey is the label key that SKS Controller attaches to the
	// underlying resources it controls.
	SKSLabelKey = GroupName + "/serverlessservice"

	// ServiceTypeKey is the label key attached to a service specifying the type of service.
	// e.g. Public, Private.
	ServiceTypeKey = GroupName + "/serviceType"

	// OriginSecretNameLabelKey is the label key attached to the TLS secret to indicate
	// the name of the origin secret that the TLS secret is copied from.
	OriginSecretNameLabelKey = GroupName + "/originSecretName"

	// OriginSecretNamespaceLabelKey is the label key attached to the TLS secret
	// to indicate the namespace of the origin secret that the TLS secret is copied from.
	OriginSecretNamespaceLabelKey = GroupName + "/originSecretNamespace"

	// CertificateClassAnnotationKey is the annotation for the
	// explicit class of Certificate that a particular resource has
	// opted into. For example,
	//
	//    networking.knative.dev/certificate.class: some-network-impl
	//
	// This uses a different domain because unlike the resource, it is
	// user-facing.
	//
	// The parent resource may use its own annotations to choose the
	// annotation value for the Certificate it uses.  Based on such
	// value a different reconciliation logic may be used (for examples,
	// Cert-Manager-based Certificate will reconcile into a Cert-Manager Certificate).
	CertificateClassAnnotationKey = "networking.knative.dev/certificate.class"

	// ActivatorServiceName is the name of the activator Kubernetes service.
	ActivatorServiceName = "activator-service"

	// DeprecatedDisableWildcardCertLabelKey is the deprecated label key attached to a namespace to indicate that
	// a wildcard certificate should be not created for it.
	DeprecatedDisableWildcardCertLabelKey = GroupName + "/disableWildcardCert"

	// DisableWildcardCertLabelKey is the label key attached to a namespace to indicate that
	// a wildcard certificate should be not created for it.
	DisableWildcardCertLabelKey = "networking.knative.dev/disableWildcardCert"

	// WildcardCertDomainLabelKey is the label key attached to a certificate to indicate the
	// domain for which it was issued.
	WildcardCertDomainLabelKey = "networking.knative.dev/wildcardDomain"
)

// ServiceType is the enumeration type for the Kubernetes services
// that we have in our system, classified by usage purpose.
type ServiceType string

const (
	// ServiceTypePrivate is the label value for internal only services
	// for user applications.
	ServiceTypePrivate ServiceType = "Private"
	// ServiceTypePublic is the label value for externally reachable
	// services for user applications.
	ServiceTypePublic ServiceType = "Public"
)

// Pseudo-constants
var (
	// DefaultRetryCount will be set if Attempts not specified.
	DefaultRetryCount = 3
)
<commit_msg>Add constant for label to disable AutoTLS (#124)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package networking

const (
	// GroupName is the name for the networking API group.
	GroupName = "networking.internal.knative.dev"

	// IngressClassAnnotationKey is the annotation for the
	// explicit class of Ingress that a particular resource has
	// opted into. For example,
	//
	//    networking.knative.dev/ingress.class: some-network-impl
	//
	// This uses a different domain because unlike the resource, it is
	// user-facing.
	//
	// The parent resource may use its own annotations to choose the
	// annotation value for the Ingress it uses.  Based on such
	// value a different reconciliation logic may be used (for examples,
	// Istio-based Ingress will reconcile into a VirtualService).
	IngressClassAnnotationKey = "networking.knative.dev/ingress.class"

	// IngressLabelKey is the label key attached to underlying network programming
	// resources to indicate which Ingress triggered their creation.
	IngressLabelKey = GroupName + "/ingress"

	// SKSLabelKey is the label key that SKS Controller attaches to the
	// underlying resources it controls.
	SKSLabelKey = GroupName + "/serverlessservice"

	// ServiceTypeKey is the label key attached to a service specifying the type of service.
	// e.g. Public, Private.
	ServiceTypeKey = GroupName + "/serviceType"

	// OriginSecretNameLabelKey is the label key attached to the TLS secret to indicate
	// the name of the origin secret that the TLS secret is copied from.
	OriginSecretNameLabelKey = GroupName + "/originSecretName"

	// OriginSecretNamespaceLabelKey is the label key attached to the TLS secret
	// to indicate the namespace of the origin secret that the TLS secret is copied from.
	OriginSecretNamespaceLabelKey = GroupName + "/originSecretNamespace"

	// CertificateClassAnnotationKey is the annotation for the
	// explicit class of Certificate that a particular resource has
	// opted into. For example,
	//
	//    networking.knative.dev/certificate.class: some-network-impl
	//
	// This uses a different domain because unlike the resource, it is
	// user-facing.
	//
	// The parent resource may use its own annotations to choose the
	// annotation value for the Certificate it uses.  Based on such
	// value a different reconciliation logic may be used (for examples,
	// Cert-Manager-based Certificate will reconcile into a Cert-Manager Certificate).
	CertificateClassAnnotationKey = "networking.knative.dev/certificate.class"

	// ActivatorServiceName is the name of the activator Kubernetes service.
	ActivatorServiceName = "activator-service"

	// DeprecatedDisableWildcardCertLabelKey is the deprecated label key attached to a namespace to indicate that
	// a wildcard certificate should be not created for it.
	DeprecatedDisableWildcardCertLabelKey = GroupName + "/disableWildcardCert"

	// DisableWildcardCertLabelKey is the label key attached to a namespace to indicate that
	// a wildcard certificate should be not created for it.
	DisableWildcardCertLabelKey = "networking.knative.dev/disableWildcardCert"

	// DisableAutoTLSLabelKey is the label key attached to a namespace to indicate that
	// AutoTLS should not be enabled for it.
	DisableAutoTLSLabelKey = "networking.knative.dev/disableAutoTLS"

	// WildcardCertDomainLabelKey is the label key attached to a certificate to indicate the
	// domain for which it was issued.
	WildcardCertDomainLabelKey = "networking.knative.dev/wildcardDomain"
)

// ServiceType is the enumeration type for the Kubernetes services
// that we have in our system, classified by usage purpose.
type ServiceType string

const (
	// ServiceTypePrivate is the label value for internal only services
	// for user applications.
	ServiceTypePrivate ServiceType = "Private"
	// ServiceTypePublic is the label value for externally reachable
	// services for user applications.
	ServiceTypePublic ServiceType = "Public"
)

// Pseudo-constants
var (
	// DefaultRetryCount will be set if Attempts not specified.
	DefaultRetryCount = 3
)
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"reflect"
	"time"

	apiv1 "github.com/kubeflow/common/pkg/apis/common/v1"
	"github.com/kubeflow/common/pkg/controller.v1/expectation"
	"github.com/kubeflow/common/pkg/core"
	commonutil "github.com/kubeflow/common/pkg/util"
	"github.com/kubeflow/common/pkg/util/k8sutil"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

func (jc *JobController) DeletePodsAndServices(runPolicy *apiv1.RunPolicy, job interface{}, pods []*v1.Pod) error {
	if len(pods) == 0 {
		return nil
	}

	// Delete nothing when the cleanPodPolicy is None.
	if *runPolicy.CleanPodPolicy == apiv1.CleanPodPolicyNone {
		return nil
	}

	for _, pod := range pods {
		// Note that pending pod will turn into running once schedulable,
		// not cleaning it may leave orphan running pod in the future,
		// we should treat it equivalent to running phase here.
		if *runPolicy.CleanPodPolicy == apiv1.CleanPodPolicyRunning && pod.Status.Phase != v1.PodRunning && pod.Status.Phase != v1.PodPending {
			continue
		}
		if err := jc.PodControl.DeletePod(pod.Namespace, pod.Name, job.(runtime.Object)); err != nil {
			return err
		}
		// Pod and service have the same name, thus the service could be deleted using pod's name.
		if err := jc.ServiceControl.DeleteService(pod.Namespace, pod.Name, job.(runtime.Object)); err != nil {
			return err
		}
	}
	return nil
}

// recordAbnormalPods records the active pod whose latest condition is not in True status.
func (jc *JobController) recordAbnormalPods(activePods []*v1.Pod, object runtime.Object) {
	core.RecordAbnormalPods(activePods, object, jc.Recorder)
}

// ReconcileJobs checks and updates replicas for each given ReplicaSpec.
// It will requeue the job in case of an error while creating/deleting pods/services.
func (jc *JobController) ReconcileJobs(
	job interface{},
	replicas map[apiv1.ReplicaType]*apiv1.ReplicaSpec,
	jobStatus apiv1.JobStatus,
	runPolicy *apiv1.RunPolicy) error {

	metaObject, ok := job.(metav1.Object)
	jobName := metaObject.GetName()
	if !ok {
		return fmt.Errorf("job is not of type metav1.Object")
	}
	runtimeObject, ok := job.(runtime.Object)
	if !ok {
		return fmt.Errorf("job is not of type runtime.Object")
	}
	jobKey, err := KeyFunc(job)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("Couldn't get key for job object %#v: %v", job, err))
		return err
	}
	// Reset expectations
	// 1. Since `ReconcileJobs` is called, we expect that previous expectations are all satisfied,
	//    and it's safe to reset the expectations
	// 2. Reset expectations can avoid dirty data such as `expectedDeletion = -1`
	//    (pod or service was deleted unexpectedly)
	jc.ResetExpectations(jobKey, replicas)

	log.Infof("Reconciling for job %s", metaObject.GetName())
	pods, err := jc.Controller.GetPodsForJob(job)
	if err != nil {
		log.Warnf("GetPodsForJob error %v", err)
		return err
	}

	services, err := jc.Controller.GetServicesForJob(job)
	if err != nil {
		log.Warnf("GetServicesForJob error %v", err)
		return err
	}

	oldStatus := jobStatus.DeepCopy()
	if commonutil.IsSucceeded(jobStatus) || commonutil.IsFailed(jobStatus) {
		// If the Job is succeed or failed, delete all pods and services.
		if err := jc.DeletePodsAndServices(runPolicy, job, pods); err != nil {
			return err
		}

		if err := jc.CleanupJob(runPolicy, jobStatus, job); err != nil {
			return err
		}

		if jc.Config.EnableGangScheduling {
			jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, "JobTerminated", "Job has been terminated. Deleting PodGroup")
			if err := jc.DeletePodGroup(metaObject); err != nil {
				jc.Recorder.Eventf(runtimeObject, v1.EventTypeWarning, "FailedDeletePodGroup", "Error deleting: %v", err)
				return err
			} else {
				jc.Recorder.Eventf(runtimeObject, v1.EventTypeNormal, "SuccessfulDeletePodGroup", "Deleted PodGroup: %v", jobName)
			}
		}

		// At this point the pods may have been deleted.
		// 1) If the job succeeded, we manually set the replica status.
		// 2) If any replicas are still active, set their status to succeeded.
		if commonutil.IsSucceeded(jobStatus) {
			for rtype := range jobStatus.ReplicaStatuses {
				jobStatus.ReplicaStatuses[rtype].Succeeded += jobStatus.ReplicaStatuses[rtype].Active
				jobStatus.ReplicaStatuses[rtype].Active = 0
			}
		}

		// No need to update the job status if the status hasn't changed since last time.
		if !reflect.DeepEqual(*oldStatus, jobStatus) {
			return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus)
		}

		return nil
	}

	// retrieve the previous number of retry
	previousRetry := jc.WorkQueue.NumRequeues(jobKey)

	activePods := k8sutil.FilterActivePods(pods)

	jc.recordAbnormalPods(activePods, runtimeObject)

	active := int32(len(activePods))
	failed := k8sutil.FilterPodCount(pods, v1.PodFailed)
	totalReplicas := k8sutil.GetTotalReplicas(replicas)
	prevReplicasFailedNum := k8sutil.GetTotalFailedReplicas(jobStatus.ReplicaStatuses)

	var failureMessage string
	jobExceedsLimit := false
	exceedsBackoffLimit := false
	pastBackoffLimit := false

	if runPolicy.BackoffLimit != nil {
		jobHasNewFailure := failed > prevReplicasFailedNum
		// new failures happen when status does not reflect the failures and active
		// is different than parallelism, otherwise the previous controller loop
		// failed updating status so even if we pick up failure it is not a new one
		exceedsBackoffLimit = jobHasNewFailure && (active != totalReplicas) &&
			(int32(previousRetry)+1 > *runPolicy.BackoffLimit)

		pastBackoffLimit, err = jc.PastBackoffLimit(jobName, runPolicy, replicas, pods)
		if err != nil {
			return err
		}
	}

	if exceedsBackoffLimit || pastBackoffLimit {
		// check if the number of pod restart exceeds backoff (for restart OnFailure only)
		// OR if the number of failed jobs increased since the last syncJob
		jobExceedsLimit = true
		failureMessage = fmt.Sprintf("Job %s has failed because it has reached the specified backoff limit", jobName)
	} else if jc.PastActiveDeadline(runPolicy, jobStatus) {
		failureMessage = fmt.Sprintf("Job %s has failed because it was active longer than specified deadline", jobName)
		jobExceedsLimit = true
	}

	if jobExceedsLimit {
		// Set job completion time before resource cleanup
		if jobStatus.CompletionTime == nil {
			now := metav1.Now()
			jobStatus.CompletionTime = &now
		}

		// If the Job exceeds backoff limit or is past active deadline
		// delete all pods and services, then set the status to failed
		if err := jc.DeletePodsAndServices(runPolicy, job, pods); err != nil {
			return err
		}

		if err := jc.CleanupJob(runPolicy, jobStatus, job); err != nil {
			return err
		}

		if jc.Config.EnableGangScheduling {
			jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, "JobTerminated", "Job has been terminated. Deleting PodGroup")
			if err := jc.DeletePodGroup(metaObject); err != nil {
				jc.Recorder.Eventf(runtimeObject, v1.EventTypeWarning, "FailedDeletePodGroup", "Error deleting: %v", err)
				return err
			} else {
				jc.Recorder.Eventf(runtimeObject, v1.EventTypeNormal, "SuccessfulDeletePodGroup", "Deleted PodGroup: %v", jobName)
			}
		}

		jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, commonutil.JobFailedReason, failureMessage)

		if err := commonutil.UpdateJobConditions(&jobStatus, apiv1.JobFailed, commonutil.JobFailedReason, failureMessage); err != nil {
			log.Infof("Append job condition error: %v", err)
			return err
		}

		return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus)
	} else {
		// General cases which need to reconcile
		if jc.Config.EnableGangScheduling {
			minMember := totalReplicas
			queue := ""
			priorityClass := ""
			var minResources *v1.ResourceList

			if runPolicy.SchedulingPolicy != nil {
				if runPolicy.SchedulingPolicy.MinAvailable != nil {
					minMember = *runPolicy.SchedulingPolicy.MinAvailable
				}

				if runPolicy.SchedulingPolicy.Queue != "" {
					queue = runPolicy.SchedulingPolicy.Queue
				}

				if runPolicy.SchedulingPolicy.PriorityClass != "" {
					priorityClass = runPolicy.SchedulingPolicy.PriorityClass
				}

				if runPolicy.SchedulingPolicy.MinResources != nil {
					minResources = runPolicy.SchedulingPolicy.MinResources
				}
			}

			if minResources == nil {
				minResources = jc.calcPGMinResources(minMember, replicas)
			}

			pgSpec := v1beta1.PodGroupSpec{
				MinMember:         minMember,
				Queue:             queue,
				PriorityClassName: priorityClass,
				MinResources:      minResources,
			}

			syncReplicas := true
			pg, err := jc.SyncPodGroup(metaObject, pgSpec)
			if err != nil {
				log.Warnf("Sync PodGroup %v: %v", jobKey, err)
				syncReplicas = false
			}

			// Delay pods creation until podgroup status is inqueue
			if pg == nil || pg.Status.Phase == "" || pg.Status.Phase == v1beta1.PodGroupPending {
				log.Warnf("PodGroup %v unschedulable", jobKey)
				syncReplicas = false
			}

			if !syncReplicas {
				now := metav1.Now()
				jobStatus.LastReconcileTime = &now

				// Update job status here to trigger a new reconciliation
				return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus)
			}
		}

		// Diff current active pods/services with replicas.
		for rtype, spec := range replicas {
			err := jc.Controller.ReconcilePods(metaObject, &jobStatus, pods, rtype, spec, replicas)
			if err != nil {
				log.Warnf("ReconcilePods error %v", err)
				return err
			}

			err = jc.Controller.ReconcileServices(metaObject, services, rtype, spec)

			if err != nil {
				log.Warnf("ReconcileServices error %v", err)
				return err
			}
		}
	}

	err = jc.Controller.UpdateJobStatus(job, replicas, &jobStatus)
	if err != nil {
		log.Warnf("UpdateJobStatus error %v", err)
		return err
	}
	// No need to update the job status if the status hasn't changed since last time.
	if !reflect.DeepEqual(*oldStatus, jobStatus) {
		return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus)
	}
	return nil
}

// ResetExpectations reset the expectation for creates and deletes of pod/service to zero.
func (jc *JobController) ResetExpectations(jobKey string, replicas map[apiv1.ReplicaType]*apiv1.ReplicaSpec) {
	for rtype := range replicas {
		expectationPodsKey := expectation.GenExpectationPodsKey(jobKey, string(rtype))
		jc.Expectations.SetExpectations(expectationPodsKey, 0, 0)
		expectationServicesKey := expectation.GenExpectationServicesKey(jobKey, string(rtype))
		jc.Expectations.SetExpectations(expectationServicesKey, 0, 0)
	}
}

// PastActiveDeadline checks if job has ActiveDeadlineSeconds field set and if it is exceeded.
func (jc *JobController) PastActiveDeadline(runPolicy *apiv1.RunPolicy, jobStatus apiv1.JobStatus) bool {
	return core.PastActiveDeadline(runPolicy, jobStatus)
}

// PastBackoffLimit checks if container restartCounts sum exceeds BackoffLimit
// this method applies only to pods when restartPolicy is one of OnFailure, Always or ExitCode
func (jc *JobController) PastBackoffLimit(jobName string, runPolicy *apiv1.RunPolicy,
	replicas map[apiv1.ReplicaType]*apiv1.ReplicaSpec, pods []*v1.Pod) (bool, error) {
	return core.PastBackoffLimit(jobName, runPolicy, replicas, pods, jc.FilterPodsForReplicaType)
}

func (jc *JobController) CleanupJob(runPolicy *apiv1.RunPolicy, jobStatus apiv1.JobStatus, job interface{}) error {
	currentTime := time.Now()
	metaObject, _ := job.(metav1.Object)
	ttl := runPolicy.TTLSecondsAfterFinished
	if ttl == nil {
		return nil
	}
	duration := time.Second * time.Duration(*ttl)
	// todo: Is the jobStatus.CompletionTime maybe nil ?
	finishTime := jobStatus.CompletionTime
	expireTime := finishTime.Add(duration)
	if currentTime.After(expireTime) {
		err := jc.Controller.DeleteJob(job)
		if err != nil {
			commonutil.LoggerForJob(metaObject).Warnf("Cleanup Job error: %v.", err)
			return err
		}
		return nil
	} else {
		if finishTime.After(currentTime) {
			commonutil.LoggerForJob(metaObject).Warnf("Found Job finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.")
		}
		remaining := expireTime.Sub(currentTime)
		key, err := KeyFunc(job)
		if err != nil {
			commonutil.LoggerForJob(metaObject).Warnf("Couldn't get key for job object: %v", err)
			return err
		}
		jc.WorkQueue.AddAfter(key, remaining)
		return nil
	}
}

func (jc *JobController) calcPGMinResources(minMember int32, replicas map[apiv1.ReplicaType]*apiv1.ReplicaSpec) *v1.ResourceList {
	return CalcPGMinResources(minMember, replicas, jc.PriorityClassLister.Get)
}
<commit_msg>feat(job): Add a check to avoid null pointer exception (#178)<commit_after>package common

import (
	"fmt"
	"reflect"
	"time"

	apiv1 "github.com/kubeflow/common/pkg/apis/common/v1"
	"github.com/kubeflow/common/pkg/controller.v1/expectation"
	"github.com/kubeflow/common/pkg/core"
	commonutil "github.com/kubeflow/common/pkg/util"
	"github.com/kubeflow/common/pkg/util/k8sutil"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"volcano.sh/apis/pkg/apis/scheduling/v1beta1"
)

func (jc *JobController) DeletePodsAndServices(runPolicy *apiv1.RunPolicy, job interface{}, pods []*v1.Pod) error {
	if len(pods) == 0 {
		return nil
	}

	// Delete nothing when the cleanPodPolicy is None.
	if *runPolicy.CleanPodPolicy == apiv1.CleanPodPolicyNone {
		return nil
	}

	for _, pod := range pods {
		// Note that pending pod will turn into running once schedulable,
		// not cleaning it may leave orphan running pod in the future,
		// we should treat it equivalent to running phase here.
		if *runPolicy.CleanPodPolicy == apiv1.CleanPodPolicyRunning && pod.Status.Phase != v1.PodRunning && pod.Status.Phase != v1.PodPending {
			continue
		}
		if err := jc.PodControl.DeletePod(pod.Namespace, pod.Name, job.(runtime.Object)); err != nil {
			return err
		}
		// Pod and service have the same name, thus the service could be deleted using pod's name.
		if err := jc.ServiceControl.DeleteService(pod.Namespace, pod.Name, job.(runtime.Object)); err != nil {
			return err
		}
	}
	return nil
}

// recordAbnormalPods records the active pod whose latest condition is not in True status.
func (jc *JobController) recordAbnormalPods(activePods []*v1.Pod, object runtime.Object) {
	core.RecordAbnormalPods(activePods, object, jc.Recorder)
}

// ReconcileJobs checks and updates replicas for each given ReplicaSpec.
// It will requeue the job in case of an error while creating/deleting pods/services.
func (jc *JobController) ReconcileJobs(
	job interface{},
	replicas map[apiv1.ReplicaType]*apiv1.ReplicaSpec,
	jobStatus apiv1.JobStatus,
	runPolicy *apiv1.RunPolicy) error {

	metaObject, ok := job.(metav1.Object)
	jobName := metaObject.GetName()
	if !ok {
		return fmt.Errorf("job is not of type metav1.Object")
	}
	runtimeObject, ok := job.(runtime.Object)
	if !ok {
		return fmt.Errorf("job is not of type runtime.Object")
	}
	jobKey, err := KeyFunc(job)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("Couldn't get key for job object %#v: %v", job, err))
		return err
	}
	// Reset expectations
	// 1. Since `ReconcileJobs` is called, we expect that previous expectations are all satisfied,
	//    and it's safe to reset the expectations
	// 2. Reset expectations can avoid dirty data such as `expectedDeletion = -1`
	//    (pod or service was deleted unexpectedly)
	jc.ResetExpectations(jobKey, replicas)

	log.Infof("Reconciling for job %s", metaObject.GetName())
	pods, err := jc.Controller.GetPodsForJob(job)
	if err != nil {
		log.Warnf("GetPodsForJob error %v", err)
		return err
	}

	services, err := jc.Controller.GetServicesForJob(job)
	if err != nil {
		log.Warnf("GetServicesForJob error %v", err)
		return err
	}

	oldStatus := jobStatus.DeepCopy()
	if commonutil.IsSucceeded(jobStatus) || commonutil.IsFailed(jobStatus) {
		// If the Job is succeed or failed, delete all pods and services.
		if err := jc.DeletePodsAndServices(runPolicy, job, pods); err != nil {
			return err
		}

		if err := jc.CleanupJob(runPolicy, jobStatus, job); err != nil {
			return err
		}

		if jc.Config.EnableGangScheduling {
			jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, "JobTerminated", "Job has been terminated. Deleting PodGroup")
			if err := jc.DeletePodGroup(metaObject); err != nil {
				jc.Recorder.Eventf(runtimeObject, v1.EventTypeWarning, "FailedDeletePodGroup", "Error deleting: %v", err)
				return err
			} else {
				jc.Recorder.Eventf(runtimeObject, v1.EventTypeNormal, "SuccessfulDeletePodGroup", "Deleted PodGroup: %v", jobName)
			}
		}

		// At this point the pods may have been deleted.
		// 1) If the job succeeded, we manually set the replica status.
		// 2) If any replicas are still active, set their status to succeeded.
		if commonutil.IsSucceeded(jobStatus) {
			for rtype := range jobStatus.ReplicaStatuses {
				jobStatus.ReplicaStatuses[rtype].Succeeded += jobStatus.ReplicaStatuses[rtype].Active
				jobStatus.ReplicaStatuses[rtype].Active = 0
			}
		}

		// No need to update the job status if the status hasn't changed since last time.
		if !reflect.DeepEqual(*oldStatus, jobStatus) {
			return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus)
		}

		return nil
	}

	// retrieve the previous number of retry
	previousRetry := jc.WorkQueue.NumRequeues(jobKey)

	activePods := k8sutil.FilterActivePods(pods)

	jc.recordAbnormalPods(activePods, runtimeObject)

	active := int32(len(activePods))
	failed := k8sutil.FilterPodCount(pods, v1.PodFailed)
	totalReplicas := k8sutil.GetTotalReplicas(replicas)
	prevReplicasFailedNum := k8sutil.GetTotalFailedReplicas(jobStatus.ReplicaStatuses)

	var failureMessage string
	jobExceedsLimit := false
	exceedsBackoffLimit := false
	pastBackoffLimit := false

	if runPolicy.BackoffLimit != nil {
		jobHasNewFailure := failed > prevReplicasFailedNum
		// new failures happen when status does not reflect the failures and active
		// is different than parallelism, otherwise the previous controller loop
		// failed updating status so even if we pick up failure it is not a new one
		exceedsBackoffLimit = jobHasNewFailure && (active != totalReplicas) &&
			(int32(previousRetry)+1 > *runPolicy.BackoffLimit)

		pastBackoffLimit, err = jc.PastBackoffLimit(jobName, runPolicy, replicas, pods)
		if err != nil {
			return err
		}
	}

	if exceedsBackoffLimit || pastBackoffLimit {
		// check if the number of pod restart exceeds backoff (for restart OnFailure only)
		// OR if the number of failed jobs increased since the last syncJob
		jobExceedsLimit = true
		failureMessage = fmt.Sprintf("Job %s has failed because it has reached the specified backoff limit", jobName)
	} else if jc.PastActiveDeadline(runPolicy, jobStatus) {
		failureMessage = fmt.Sprintf("Job %s has failed because it was active longer than specified deadline", jobName)
		jobExceedsLimit = true
	}

	if jobExceedsLimit {
		// Set job completion time before resource cleanup
		if jobStatus.CompletionTime == nil {
			now := metav1.Now()
			jobStatus.CompletionTime = &now
		}

		// If the Job exceeds backoff limit or is past active deadline
		// delete all pods and services, then set the status to failed
		if err := jc.DeletePodsAndServices(runPolicy, job, pods); err != nil {
			return err
		}

		if err := jc.CleanupJob(runPolicy, jobStatus, job); err != nil {
			return err
		}

		if jc.Config.EnableGangScheduling {
			jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, "JobTerminated", "Job has been terminated. Deleting PodGroup")
			if err := jc.DeletePodGroup(metaObject); err != nil {
				jc.Recorder.Eventf(runtimeObject, v1.EventTypeWarning, "FailedDeletePodGroup", "Error deleting: %v", err)
				return err
			} else {
				jc.Recorder.Eventf(runtimeObject, v1.EventTypeNormal, "SuccessfulDeletePodGroup", "Deleted PodGroup: %v", jobName)
			}
		}

		jc.Recorder.Event(runtimeObject, v1.EventTypeNormal, commonutil.JobFailedReason, failureMessage)

		if err := commonutil.UpdateJobConditions(&jobStatus, apiv1.JobFailed, commonutil.JobFailedReason, failureMessage); err != nil {
			log.Infof("Append job condition error: %v", err)
			return err
		}

		return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus)
	} else {
		// General cases which need to reconcile
		if jc.Config.EnableGangScheduling {
			minMember := totalReplicas
			queue := ""
			priorityClass := ""
			var minResources *v1.ResourceList

			if runPolicy.SchedulingPolicy != nil {
				if runPolicy.SchedulingPolicy.MinAvailable != nil {
					minMember = *runPolicy.SchedulingPolicy.MinAvailable
				}

				if runPolicy.SchedulingPolicy.Queue != "" {
					queue = runPolicy.SchedulingPolicy.Queue
				}

				if runPolicy.SchedulingPolicy.PriorityClass != "" {
					priorityClass = runPolicy.SchedulingPolicy.PriorityClass
				}

				if runPolicy.SchedulingPolicy.MinResources != nil {
					minResources = runPolicy.SchedulingPolicy.MinResources
				}
			}

			if minResources == nil {
				minResources = jc.calcPGMinResources(minMember, replicas)
			}

			pgSpec := v1beta1.PodGroupSpec{
				MinMember:         minMember,
				Queue:             queue,
				PriorityClassName: priorityClass,
				MinResources:      minResources,
			}

			syncReplicas := true
			pg, err := jc.SyncPodGroup(metaObject, pgSpec)
			if err != nil {
				log.Warnf("Sync PodGroup %v: %v", jobKey, err)
				syncReplicas = false
			}

			// Delay pods creation until podgroup status is inqueue
			if pg == nil || pg.Status.Phase == "" || pg.Status.Phase == v1beta1.PodGroupPending {
				log.Warnf("PodGroup %v unschedulable", jobKey)
				syncReplicas = false
			}

			if !syncReplicas {
				now := metav1.Now()
				jobStatus.LastReconcileTime = &now

				// Update job status here to trigger a new reconciliation
				return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus)
			}
		}

		// Diff current active pods/services with replicas.
		for rtype, spec := range replicas {
			err := jc.Controller.ReconcilePods(metaObject, &jobStatus, pods, rtype, spec, replicas)
			if err != nil {
				log.Warnf("ReconcilePods error %v", err)
				return err
			}

			err = jc.Controller.ReconcileServices(metaObject, services, rtype, spec)

			if err != nil {
				log.Warnf("ReconcileServices error %v", err)
				return err
			}
		}
	}

	err = jc.Controller.UpdateJobStatus(job, replicas, &jobStatus)
	if err != nil {
		log.Warnf("UpdateJobStatus error %v", err)
		return err
	}
	// No need to update the job status if the status hasn't changed since last time.
	if !reflect.DeepEqual(*oldStatus, jobStatus) {
		return jc.Controller.UpdateJobStatusInApiServer(job, &jobStatus)
	}
	return nil
}

// ResetExpectations reset the expectation for creates and deletes of pod/service to zero.
func (jc *JobController) ResetExpectations(jobKey string, replicas map[apiv1.ReplicaType]*apiv1.ReplicaSpec) {
	for rtype := range replicas {
		expectationPodsKey := expectation.GenExpectationPodsKey(jobKey, string(rtype))
		jc.Expectations.SetExpectations(expectationPodsKey, 0, 0)
		expectationServicesKey := expectation.GenExpectationServicesKey(jobKey, string(rtype))
		jc.Expectations.SetExpectations(expectationServicesKey, 0, 0)
	}
}

// PastActiveDeadline checks if job has ActiveDeadlineSeconds field set and if it is exceeded.
func (jc *JobController) PastActiveDeadline(runPolicy *apiv1.RunPolicy, jobStatus apiv1.JobStatus) bool {
	return core.PastActiveDeadline(runPolicy, jobStatus)
}

// PastBackoffLimit checks if container restartCounts sum exceeds BackoffLimit
// this method applies only to pods when restartPolicy is one of OnFailure, Always or ExitCode
func (jc *JobController) PastBackoffLimit(jobName string, runPolicy *apiv1.RunPolicy,
	replicas map[apiv1.ReplicaType]*apiv1.ReplicaSpec, pods []*v1.Pod) (bool, error) {
	return core.PastBackoffLimit(jobName, runPolicy, replicas, pods, jc.FilterPodsForReplicaType)
}

func (jc *JobController) CleanupJob(runPolicy *apiv1.RunPolicy, jobStatus apiv1.JobStatus, job interface{}) error {
	currentTime := time.Now()
	metaObject, _ := job.(metav1.Object)
	ttl := runPolicy.TTLSecondsAfterFinished
	if ttl == nil {
		return nil
	}
	duration := time.Second * time.Duration(*ttl)
	if jobStatus.CompletionTime == nil {
		return fmt.Errorf("job completion time is nil, cannot cleanup")
	}
	finishTime := jobStatus.CompletionTime
	expireTime := finishTime.Add(duration)
	if currentTime.After(expireTime) {
		err := jc.Controller.DeleteJob(job)
		if err != nil {
			commonutil.LoggerForJob(metaObject).Warnf("Cleanup Job error: %v.", err)
			return err
		}
		return nil
	} else {
		if finishTime.After(currentTime) {
			commonutil.LoggerForJob(metaObject).Warnf("Found Job finished in the future. This is likely due to time skew in the cluster. Job cleanup will be deferred.")
		}
		remaining := expireTime.Sub(currentTime)
		key, err := KeyFunc(job)
		if err != nil {
			commonutil.LoggerForJob(metaObject).Warnf("Couldn't get key for job object: %v", err)
			return err
		}
		jc.WorkQueue.AddAfter(key, remaining)
		return nil
	}
}

func (jc *JobController) calcPGMinResources(minMember int32, replicas map[apiv1.ReplicaType]*apiv1.ReplicaSpec) *v1.ResourceList {
	return CalcPGMinResources(minMember, replicas, jc.PriorityClassLister.Get)
}
<|endoftext|>


<commit_before>package dory

import (
	"bytes"
	"math/rand"
	"testing"
)

const (
	minKeySize = 1
	maxKeySize = 32
	minValSize = 1
	maxValSize = 128 * 1024

	bufferSize = 16 * 1024 * 1024
)

var (
	values = make(map[string][]byte)

	shortKey = make([]byte, 8)
	longKey  = make([]byte, 1024)
)

func init() {
	const generatedValues = 1024

	for i := 0; i < generatedValues; i++ {
		keyLen := rand.Intn(maxKeySize-minKeySize) + minKeySize
		valLen := rand.Intn(maxValSize-minValSize) + minValSize

		keyBuf := make([]byte, keyLen)
		rand.Read(keyBuf)
		valBuf := make([]byte, valLen)
		rand.Read(valBuf)
		values[string(keyBuf)] = valBuf
	}

	rand.Read(shortKey)
	rand.Read(longKey)
}

func checkSpace(t *testing.T, pt *PackedTable) {
	t.Helper()
	if pt.LiveSpace()+pt.FreeSpace()+pt.DeletedSpace() != bufferSize {
		t.Errorf("LiveSpace %d + FreeSpace %d + DeletedSpace %d != %d",
			pt.LiveSpace(), pt.FreeSpace(), pt.DeletedSpace(), bufferSize)
	}
}

func TestPackedTable(t *testing.T) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)

	addedValues := make(map[string]bool)
	addedSize := 0
	for k, v := range values {
		err := buffer.Put([]byte(k), v)
		if err == nil {
			addedValues[k] = true
			addedSize += len(v) + len(k) + 4
		}
		checkSpace(t, buffer)
	}

	t.Logf("Added %d entries, size %d", len(addedValues), addedSize)
	if addedSize < bufferSize-maxValSize-maxKeySize-4 {
		t.Errorf("Added size %d is to small", addedSize)
	}

	// Delete ~20% of added values
	d := 0
	deletedSize := 0
	for k := range addedValues {
		if rand.Float32() > 0.2 {
			continue
		}
		deletedSize += buffer.EntrySize([]byte(k), values[k])
		buffer.Delete([]byte(k))
		delete(addedValues, k)
		d++
		checkSpace(t, buffer)
	}
	t.Logf("Deleted %d values of %d bytes", d, deletedSize)

	for i := 0; i < 2; i++ {
		for k, v := range values {
			exists := addedValues[k]

			has := buffer.Has([]byte(k))
			if has != exists {
				t.Errorf("exists %v != has %v", exists, has)
			}

			buf := buffer.Get([]byte(k))
			if exists != (buf != nil) {
				t.Errorf("exists %v != (buf %v != nil)", exists, buf)
			}
			if buf != nil && bytes.Compare(buf, v) != 0 {
				t.Errorf("buffer for %s != expected", k)
			}
		}

		if i == 0 {
			oldFreeSpace := buffer.FreeSpace()
			buffer.GC()
			if buffer.DeletedSpace() != 0 {
				t.Errorf("buffer.DeletedSpace() %d != 0", buffer.DeletedSpace())
			}
			if buffer.FreeSpace() != oldFreeSpace+deletedSize {
				t.Errorf("buffer.FreeSpace() %d != oldFreeSpace %d + deletedSize %d",
					buffer.FreeSpace(), oldFreeSpace, deletedSize)
			}
			checkSpace(t, buffer)
			t.Logf("New free space %d bytes", buffer.FreeSpace())
		}
	}
}

func TestPackedTableOverwrite(t *testing.T) {
	key := []byte("foo")
	val1 := []byte("hello")
	val2 := []byte("world")

	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	if has := buffer.Has(key); has {
		t.Errorf("Unexpected has")
	}
	if err := buffer.Put(key, val1); err != nil {
		t.Errorf("Unexpected put error %v", err)
	}
	if has := buffer.Has(key); !has {
		t.Errorf("Unexpected not has")
	}
	buf := buffer.Get(key)
	if !bytes.Equal(buf, val1) {
		t.Errorf("Unexpected get result %s", string(buf))
	}
	if err := buffer.Put(key, val2); err != nil {
		t.Errorf("Unexpected put error %v", err)
	}
	if has := buffer.Has(key); !has {
		t.Errorf("Unexpected not has")
	}
	buf = buffer.Get(key)
	if !bytes.Equal(buf, val2) {
		t.Errorf("Unexpected get result %s", string(buf))
	}
	buffer.Delete(key)
	if has := buffer.Has(key); has {
		t.Errorf("Unexpected has")
	}
}

func TestPackedTableOverwriteGC(t *testing.T) {
	key := []byte("dkjfhkdjdfhd")
	val := []byte("dfjhgkfdjghkfdj hkdfjhdfkjhgfdkhdfk")

	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	for i := 0; i < 1000000; i++ {
		// Without GC, this will fail after a while.
		if err := buffer.Put(key, val); err != nil {
			t.Fatalf("Unexpected put error %v", err)
		}
		buffer.GC()
		buf := buffer.Get(key)
		if bytes.Compare(buf, val) != 0 {
			t.Errorf("Unexpected get result %s", string(buf))
		}
	}
}

func TestPackedTableOverwriteAutoGC(t *testing.T) {
	key := []byte("dkjfhkdjdfhd")
	val := []byte("dfjhgkfdjghkfdj hkdfjhdfkjhgfdkhdfk")

	buffer := NewPackedTable(make([]byte, bufferSize), bufferSize/2)
	for i := 0; i < 1000000; i++ {
		// Without GC, this will fail after a while.
		if err := buffer.Put(key, val); err != nil {
			t.Fatalf("Unexpected put error %v", err)
		}
		buf := buffer.Get(key)
		if bytes.Compare(buf, val) != 0 {
			t.Errorf("Unexpected get result %s", string(buf))
		}
	}
}

func TestPackedTableReset(t *testing.T) {
	key := []byte("foo")
	val1 := []byte("hello")

	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	if has := buffer.Has(key); has {
		t.Errorf("Unexpected has")
	}
	if err := buffer.Put(key, val1); err != nil {
		t.Errorf("Unexpected put error %v", err)
	}
	if has := buffer.Has(key); !has {
		t.Errorf("Unexpected not has")
	}
	buf := buffer.Get(key)
	if !bytes.Equal(buf, val1) {
		t.Errorf("Unexpected get result %s", string(buf))
	}
	if buffer.NumEntries() != 1 || buffer.NumDeleted() != 0 || buffer.FreeSpace() == bufferSize {
		t.Errorf("Unexpected stats")
	}

	buffer.Reset()
	if has := buffer.Has(key); has {
		t.Errorf("Unexpected has")
	}
	if buffer.NumEntries() != 0 || buffer.NumDeleted() != 0 || buffer.FreeSpace() != bufferSize {
		t.Errorf("Unexpected stats")
	}
}

func BenchmarkPackedTableHas(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := shortKey
	val := []byte("foo")
	buffer.Put(key, val)

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Has(key)
	}
}

func BenchmarkPackedTableHasLongKey(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := longKey
	val := []byte("foo")
	buffer.Put(key, val)

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Has(key)
	}
}

func BenchmarkPackedTableHasNotExist(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := shortKey

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Has(key)
	}
}

func BenchmarkPackedTableGet(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := shortKey
	val := []byte("foo")
	buffer.Put(key, val)

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Get(key)
	}
}

func BenchmarkPackedTableGetNotExist(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := shortKey

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Get(key)
	}
}

var (
	benchmarkKeys [][]byte
	benchmarkVal  = []byte("foo")
)

func init() {
	// These should all fit into the test table.
	const generatedKeys = 4096
	// Use short keys to benchmark overhead rather than the memcpy of key/value
	// data.
	const keyLen = 8

	benchmarkKeys = make([][]byte, 0, generatedKeys)
	for i := 0; i < generatedKeys; i++ {
		keyBuf := make([]byte, keyLen)
		rand.Read(keyBuf)
		benchmarkKeys = append(benchmarkKeys, keyBuf)
	}
}

func BenchmarkPackedTablePut(b *testing.B) {
	buf := make([]byte, bufferSize)

	b.ReportAllocs()
	b.ResetTimer()
	i := 0
	table := NewPackedTable(buf, 0)
	for i < b.N {
		table.Reset()
		for _, k := range benchmarkKeys {
			if i >= b.N {
				break
			}
			err := table.Put(k, benchmarkVal)
			if err != nil {
				// 100% of entries should fit into the table.
				panic(err)
			}
			i++
		}
	}
}

func BenchmarkPackedTablePutAndDelete(b *testing.B) {
	buf := make([]byte, bufferSize)

	b.ReportAllocs()
	b.ResetTimer()
	i := 0
	table := NewPackedTable(buf, 0)
	for i < b.N {
		table.Reset()
		for _, k := range benchmarkKeys {
			if i >= b.N {
				break
			}
			err := table.Put(k, benchmarkVal)
			if err != nil {
				panic(err)
			}
			i++
		}
		for _, k := range benchmarkKeys {
			table.Delete(k)
		}
	}
}

func BenchmarkPackedTablePutAndOverwrite(b *testing.B) {
	buf := make([]byte, bufferSize)

	b.ReportAllocs()
	b.ResetTimer()
	i := 0
	table := NewPackedTable(buf, 0)
	for i < b.N {
		table.Reset()
		for _, k := range benchmarkKeys {
			if i >= b.N {
				break
			}
			err := table.Put(k, benchmarkVal)
			if err != nil {
				panic(err)
			}
			i++
		}
		for _, k := range benchmarkKeys {
			err := table.Put(k, benchmarkVal)
			if err != nil {
				panic(err)
			}
		}
	}
}

func BenchmarkPackedTableGC(b *testing.B) {
	buf := make([]byte, bufferSize)

	var deletedKeys [][]byte
	for _, k := range benchmarkKeys {
		if rand.Float32() > 0.2 {
			continue
		}
		deletedKeys = append(deletedKeys, k)
	}

	b.ReportAllocs()
	b.ResetTimer()
	i := 0
	table := NewPackedTable(buf, 0)
	for i < b.N {
		b.StopTimer()
		table.Reset()
		for _, k := range benchmarkKeys {
			err := table.Put(k, benchmarkVal)
			if err != nil {
				panic(err)
			}
		}
		for _, k := range deletedKeys {
			table.Delete(k)
		}
		b.StartTimer()
		table.GC()
		// An "op" is a GC'd key. So the repoerted ns/op should be interpreted
		// as GCns/key. A GC with 1234 active keys should take 1234*ns/op.
		i += table.NumEntries()
	}
}
<commit_msg>Benchmark operations with multiple key sizes.<commit_after>package dory

import (
	"bytes"
	"math/rand"
	"testing"
)

const (
	minKeySize = 1
	maxKeySize = 32
	minValSize = 1
	maxValSize = 128 * 1024

	bufferSize = 16 * 1024 * 1024
)

var (
	values = make(map[string][]byte)

	shortKey = make([]byte, 8)
	longKey  = make([]byte, 1024)
)

func init() {
	const generatedValues = 1024

	for i := 0; i < generatedValues; i++ {
		keyLen := rand.Intn(maxKeySize-minKeySize) + minKeySize
		valLen := rand.Intn(maxValSize-minValSize) + minValSize

		keyBuf := make([]byte, keyLen)
		rand.Read(keyBuf)
		valBuf := make([]byte, valLen)
		rand.Read(valBuf)
		values[string(keyBuf)] = valBuf
	}

	rand.Read(shortKey)
	rand.Read(longKey)
}

func checkSpace(t *testing.T, pt *PackedTable) {
	t.Helper()
	if pt.LiveSpace()+pt.FreeSpace()+pt.DeletedSpace() != bufferSize {
		t.Errorf("LiveSpace %d + FreeSpace %d + DeletedSpace %d != %d",
			pt.LiveSpace(), pt.FreeSpace(), pt.DeletedSpace(), bufferSize)
	}
}

func TestPackedTable(t *testing.T) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)

	addedValues := make(map[string]bool)
	addedSize := 0
	for k, v := range values {
		err := buffer.Put([]byte(k), v)
		if err == nil {
			addedValues[k] = true
			addedSize += len(v) + len(k) + 4
		}
		checkSpace(t, buffer)
	}

	t.Logf("Added %d entries, size %d", len(addedValues), addedSize)
	if addedSize < bufferSize-maxValSize-maxKeySize-4 {
		t.Errorf("Added size %d is to small", addedSize)
	}

	// Delete ~20% of added values
	d := 0
	deletedSize := 0
	for k := range addedValues {
		if rand.Float32() > 0.2 {
			continue
		}
		deletedSize += buffer.EntrySize([]byte(k), values[k])
		buffer.Delete([]byte(k))
		delete(addedValues, k)
		d++
		checkSpace(t, buffer)
	}
	t.Logf("Deleted %d values of %d bytes", d, deletedSize)

	for i := 0; i < 2; i++ {
		for k, v := range values {
			exists := addedValues[k]

			has := buffer.Has([]byte(k))
			if has != exists {
				t.Errorf("exists %v != has %v", exists, has)
			}

			buf := buffer.Get([]byte(k))
			if exists != (buf != nil) {
				t.Errorf("exists %v != (buf %v != nil)", exists, buf)
			}
			if buf != nil && bytes.Compare(buf, v) != 0 {
				t.Errorf("buffer for %s != expected", k)
			}
		}

		if i == 0 {
			oldFreeSpace := buffer.FreeSpace()
			buffer.GC()
			if buffer.DeletedSpace() != 0 {
				t.Errorf("buffer.DeletedSpace() %d != 0", buffer.DeletedSpace())
			}
			if buffer.FreeSpace() != oldFreeSpace+deletedSize {
				t.Errorf("buffer.FreeSpace() %d != oldFreeSpace %d + deletedSize %d",
					buffer.FreeSpace(), oldFreeSpace, deletedSize)
			}
			checkSpace(t, buffer)
			t.Logf("New free space %d bytes", buffer.FreeSpace())
		}
	}
}

func TestPackedTableOverwrite(t *testing.T) {
	key := []byte("foo")
	val1 := []byte("hello")
	val2 := []byte("world")

	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	if has := buffer.Has(key); has {
		t.Errorf("Unexpected has")
	}
	if err := buffer.Put(key, val1); err != nil {
		t.Errorf("Unexpected put error %v", err)
	}
	if has := buffer.Has(key); !has {
		t.Errorf("Unexpected not has")
	}
	buf := buffer.Get(key)
	if !bytes.Equal(buf, val1) {
		t.Errorf("Unexpected get result %s", string(buf))
	}
	if err := buffer.Put(key, val2); err != nil {
		t.Errorf("Unexpected put error %v", err)
	}
	if has := buffer.Has(key); !has {
		t.Errorf("Unexpected not has")
	}
	buf = buffer.Get(key)
	if !bytes.Equal(buf, val2) {
		t.Errorf("Unexpected get result %s", string(buf))
	}
	buffer.Delete(key)
	if has := buffer.Has(key); has {
		t.Errorf("Unexpected has")
	}
}

func TestPackedTableOverwriteGC(t *testing.T) {
	key := []byte("dkjfhkdjdfhd")
	val := []byte("dfjhgkfdjghkfdj hkdfjhdfkjhgfdkhdfk")

	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	for i := 0; i < 1000000; i++ {
		// Without GC, this will fail after a while.
		if err := buffer.Put(key, val); err != nil {
			t.Fatalf("Unexpected put error %v", err)
		}
		buffer.GC()
		buf := buffer.Get(key)
		if bytes.Compare(buf, val) != 0 {
			t.Errorf("Unexpected get result %s", string(buf))
		}
	}
}

func TestPackedTableOverwriteAutoGC(t *testing.T) {
	key := []byte("dkjfhkdjdfhd")
	val := []byte("dfjhgkfdjghkfdj hkdfjhdfkjhgfdkhdfk")

	buffer := NewPackedTable(make([]byte, bufferSize), bufferSize/2)
	for i := 0; i < 1000000; i++ {
		// Without GC, this will fail after a while.
		if err := buffer.Put(key, val); err != nil {
			t.Fatalf("Unexpected put error %v", err)
		}
		buf := buffer.Get(key)
		if bytes.Compare(buf, val) != 0 {
			t.Errorf("Unexpected get result %s", string(buf))
		}
	}
}

func TestPackedTableReset(t *testing.T) {
	key := []byte("foo")
	val1 := []byte("hello")

	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	if has := buffer.Has(key); has {
		t.Errorf("Unexpected has")
	}
	if err := buffer.Put(key, val1); err != nil {
		t.Errorf("Unexpected put error %v", err)
	}
	if has := buffer.Has(key); !has {
		t.Errorf("Unexpected not has")
	}
	buf := buffer.Get(key)
	if !bytes.Equal(buf, val1) {
		t.Errorf("Unexpected get result %s", string(buf))
	}
	if buffer.NumEntries() != 1 || buffer.NumDeleted() != 0 || buffer.FreeSpace() == bufferSize {
		t.Errorf("Unexpected stats")
	}

	buffer.Reset()
	if has := buffer.Has(key); has {
		t.Errorf("Unexpected has")
	}
	if buffer.NumEntries() != 0 || buffer.NumDeleted() != 0 || buffer.FreeSpace() != bufferSize {
		t.Errorf("Unexpected stats")
	}
}

func BenchmarkPackedTableHas(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := shortKey
	val := []byte("foo")
	buffer.Put(key, val)

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Has(key)
	}
}

func BenchmarkPackedTableHasLongKey(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := longKey
	val := []byte("foo")
	buffer.Put(key, val)

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Has(key)
	}
}

func BenchmarkPackedTableHasNotExist(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := shortKey

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Has(key)
	}
}

func BenchmarkPackedTableGet(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := shortKey
	val := []byte("foo")
	buffer.Put(key, val)

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Get(key)
	}
}

func BenchmarkPackedTableGetNotExist(b *testing.B) {
	buffer := NewPackedTable(make([]byte, bufferSize), 0)
	key := shortKey

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buffer.Get(key)
	}
}

const (
	benchKeys   = 4096
	benchKeyLen = 8
)

var (
	benchmarkVal = []byte("foo")
)

func genKeys(keyLen, numKeys int) [][]byte {
	keys := make([][]byte, 0, numKeys)
	for i := 0; i < numKeys; i++ {
		keyBuf := make([]byte, keyLen)
		rand.Read(keyBuf)
		keys = append(keys, keyBuf)
	}
	return keys
}

func benchmarkPackedTablePut_N(b *testing.B, keyLen int) {
	buf := make([]byte, bufferSize)
	benchmarkKeys := genKeys(keyLen, benchKeys)

	b.ReportAllocs()
	b.ResetTimer()
	i := 0
	table := NewPackedTable(buf, 0)
	for i < b.N {
		table.Reset()
		for _, k := range benchmarkKeys {
			if i >= b.N {
				break
			}
			err := table.Put(k, benchmarkVal)
			if err != nil {
				// 100% of entries should fit into the table.
				panic(err)
			}
			i++
		}
	}
}

func BenchmarkPackedTablePut_8(b *testing.B) {
	benchmarkPackedTablePut_N(b, 8)
}

func BenchmarkPackedTablePut_64(b *testing.B) {
	benchmarkPackedTablePut_N(b, 64)
}
func BenchmarkPackedTablePut_1024(b *testing.B) {
	benchmarkPackedTablePut_N(b, 1024)
}

func BenchmarkPackedTablePutAndOverwrite(b *testing.B) {
	buf := make([]byte, bufferSize)
	benchmarkKeys := genKeys(benchKeyLen, benchKeys)

	b.ReportAllocs()
	b.ResetTimer()
	i := 0
	table := NewPackedTable(buf, 0)
	for ; i < b.N; i += benchKeys {
		table.Reset()
		for _, k := range benchmarkKeys {
			err := table.Put(k, benchmarkVal)
			if err != nil {
				panic(err)
			}
		}
		for _, k := range benchmarkKeys {
			err := table.Put(k, benchmarkVal)
			if err != nil {
				panic(err)
			}
		}
	}
}

func benchmarkPackedTableDelete_N(b *testing.B, keyLen int) {
	buf := make([]byte, bufferSize)
	benchmarkKeys := genKeys(keyLen, benchKeys)

	b.ReportAllocs()
	b.ResetTimer()
	i := 0
	table := NewPackedTable(buf, 0)
	for ; i < b.N; i += benchKeys {
		b.StopTimer()
		table.Reset()
		for _, k := range benchmarkKeys {
			err := table.Put(k, benchmarkVal)
			if err != nil {
				panic(err)
			}
		}
		b.StartTimer()
		for _, k := range benchmarkKeys {
			table.Delete(k)
		}
	}
}

func BenchmarkPackedTableDelete_8(b *testing.B) {
	benchmarkPackedTableDelete_N(b, 8)
}

func BenchmarkPackedTableDelete_64(b *testing.B) {
	benchmarkPackedTableDelete_N(b, 64)
}

func BenchmarkPackedTableDelete_1024(b *testing.B) {
	benchmarkPackedTableDelete_N(b, 1024)
}

func benchmarkPackedTableGC_N(b *testing.B, keyLen int) {
	buf := make([]byte, bufferSize)
	benchmarkKeys := genKeys(keyLen, benchKeys)

	var deletedKeys [][]byte
	for _, k := range benchmarkKeys {
		if rand.Float32() > 0.2 {
			continue
		}
		deletedKeys = append(deletedKeys, k)
	}

	b.ReportAllocs()
	b.ResetTimer()
	i := 0
	table := NewPackedTable(buf, 0)
	for i < b.N {
		b.StopTimer()
		table.Reset()
		for _, k := range benchmarkKeys {
			err := table.Put(k, benchmarkVal)
			if err != nil {
				panic(err)
			}
		}
		for _, k := range deletedKeys {
			table.Delete(k)
		}
		b.StartTimer()
		table.GC()
		// An "op" is a GC'd key. So the reported ns/op should be interpreted
		// as GCns/key. A GC with 1234 active keys should take 1234*ns/op.
		i += table.NumEntries()
	}
}

func BenchmarkPackedTableGC_8(b *testing.B) {
	benchmarkPackedTableGC_N(b, 8)
}

func BenchmarkPackedTableGC_64(b *testing.B) {
	benchmarkPackedTableGC_N(b, 64)
}

func BenchmarkPackedTableGC_1024(b *testing.B) {
	benchmarkPackedTableGC_N(b, 1024)
}
<|endoftext|>