<commit_before>package mariadb

import (
	"crypto/sha512"
	"encoding/hex"
	"io"
	"net/http"

	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/httphelper"
	"github.com/flynn/flynn/pkg/sirenia/client"
	"github.com/flynn/flynn/pkg/sirenia/state"
	"github.com/flynn/flynn/pkg/status"
	"github.com/julienschmidt/httprouter"
	"gopkg.in/inconshreveable/log15.v2"
)

// Handler represents an HTTP API handler for the process.
type Handler struct {
	router *httprouter.Router

	Process     *Process
	Peer        *state.Peer
	Heartbeater discoverd.Heartbeater
	Logger      log15.Logger
}

// NewHandler returns a new instance of Handler.
func NewHandler() *Handler {
	h := &Handler{
		router: httprouter.New(),
		Logger: log15.New(),
	}
	h.router.Handler("GET", status.Path, status.Handler(h.healthStatus))
	h.router.GET("/status", h.handleGetStatus)
	h.router.GET("/backup", h.handleGetBackup)
	h.router.POST("/stop", h.handlePostStop)
	return h
}

// ServeHTTP serves an HTTP request.
func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { h.router.ServeHTTP(w, req) }

// healthStatus returns the current status of the peer and process.
func (h *Handler) healthStatus() status.Status {
	info := h.Peer.Info()
	if info.State == nil || info.RetryPending != nil ||
		(info.Role != state.RolePrimary && info.Role != state.RoleSync && info.Role != state.RoleAsync) {
		return status.Unhealthy
	}

	process, err := h.Process.Info()
	if err != nil || !process.Running || !process.UserExists {
		return status.Unhealthy
	} else if info.Role == state.RolePrimary {
		if !process.ReadWrite {
			return status.Unhealthy
		}
		if !info.State.Singleton && (process.SyncedDownstream == nil || info.State.Sync == nil || info.State.Sync.ID != process.SyncedDownstream.ID) {
			return status.Unhealthy
		}
	}

	return status.Healthy
}

// handleGetStatus handles request to GET /status.
func (h *Handler) handleGetStatus(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	logger := h.Logger.New("fn", "handleGetStatus")

	var status client.Status
	if h.Peer != nil {
		status.Peer = h.Peer.Info()
	}

	info, err := h.Process.Info()
	if err != nil {
		// Log the error, but don't return a 500. We will always have some
		// information to return, but MariaDB may not be online.
		logger.Error("error getting process info", "err", err)
	}
	status.Database = info

	httphelper.JSON(w, 200, &status)
}

// handleGetBackup handles request to GET /backup.
func (h *Handler) handleGetBackup(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	r, err := h.Process.Backup()
	if err != nil {
		httphelper.Error(w, err)
		return
	}
	defer r.Close()

	w.Header().Set("Trailer", backupChecksumTrailer)
	w.WriteHeader(200)

	hash := sha512.New()
	if _, err := io.Copy(io.MultiWriter(w, hash), r); err != nil {
		h.Logger.Error("error streaming backup", "err", err)
		r.Close()
		return
	}

	// Close the reader and confirm that it finished.
	// Sets a trailer at the end of the HTTP response.
	if err := r.Close(); err == nil {
		// Temporary hack for writing trailers until Flynn uses Go 1.5.
		w.(http.Flusher).Flush()
		conn, buf, _ := w.(http.Hijacker).Hijack()
		buf.WriteString("0\r\n") // eof
		http.Header{backupChecksumTrailer: {hex.EncodeToString(hash.Sum(nil))}}.Write(buf)
		buf.WriteString("\r\n") // end of trailers
		buf.Flush()
		conn.Close()
	} else {
		h.Logger.Error("error returned from backup", "err", err)
	}
}

// handlePostStop handles request to POST /stop.
func (h *Handler) handlePostStop(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	if err := h.Heartbeater.Close(); err != nil {
		httphelper.Error(w, err)
		return
	}
	if err := h.Peer.Stop(); err != nil {
		httphelper.Error(w, err)
		return
	}
	w.WriteHeader(200)
}

const backupChecksumTrailer = "Flynn-Backup-Checksum"
<commit_msg>appliance/mariadb: Stop peer before heartbeater<commit_after>package mariadb

import (
	"crypto/sha512"
	"encoding/hex"
	"io"
	"net/http"

	"github.com/flynn/flynn/discoverd/client"
	"github.com/flynn/flynn/pkg/httphelper"
	"github.com/flynn/flynn/pkg/sirenia/client"
	"github.com/flynn/flynn/pkg/sirenia/state"
	"github.com/flynn/flynn/pkg/status"
	"github.com/julienschmidt/httprouter"
	"gopkg.in/inconshreveable/log15.v2"
)

// Handler represents an HTTP API handler for the process.
type Handler struct {
	router *httprouter.Router

	Process     *Process
	Peer        *state.Peer
	Heartbeater discoverd.Heartbeater
	Logger      log15.Logger
}

// NewHandler returns a new instance of Handler.
func NewHandler() *Handler {
	h := &Handler{
		router: httprouter.New(),
		Logger: log15.New(),
	}
	h.router.Handler("GET", status.Path, status.Handler(h.healthStatus))
	h.router.GET("/status", h.handleGetStatus)
	h.router.GET("/backup", h.handleGetBackup)
	h.router.POST("/stop", h.handlePostStop)
	return h
}

// ServeHTTP serves an HTTP request.
func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) { h.router.ServeHTTP(w, req) }

// healthStatus returns the current status of the peer and process.
func (h *Handler) healthStatus() status.Status {
	info := h.Peer.Info()
	if info.State == nil || info.RetryPending != nil ||
		(info.Role != state.RolePrimary && info.Role != state.RoleSync && info.Role != state.RoleAsync) {
		return status.Unhealthy
	}

	process, err := h.Process.Info()
	if err != nil || !process.Running || !process.UserExists {
		return status.Unhealthy
	} else if info.Role == state.RolePrimary {
		if !process.ReadWrite {
			return status.Unhealthy
		}
		if !info.State.Singleton && (process.SyncedDownstream == nil || info.State.Sync == nil || info.State.Sync.ID != process.SyncedDownstream.ID) {
			return status.Unhealthy
		}
	}

	return status.Healthy
}

// handleGetStatus handles request to GET /status.
func (h *Handler) handleGetStatus(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	logger := h.Logger.New("fn", "handleGetStatus")

	var status client.Status
	if h.Peer != nil {
		status.Peer = h.Peer.Info()
	}

	info, err := h.Process.Info()
	if err != nil {
		// Log the error, but don't return a 500. We will always have some
		// information to return, but MariaDB may not be online.
		logger.Error("error getting process info", "err", err)
	}
	status.Database = info

	httphelper.JSON(w, 200, &status)
}

// handleGetBackup handles request to GET /backup.
func (h *Handler) handleGetBackup(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	r, err := h.Process.Backup()
	if err != nil {
		httphelper.Error(w, err)
		return
	}
	defer r.Close()

	w.Header().Set("Trailer", backupChecksumTrailer)
	w.WriteHeader(200)

	hash := sha512.New()
	if _, err := io.Copy(io.MultiWriter(w, hash), r); err != nil {
		h.Logger.Error("error streaming backup", "err", err)
		r.Close()
		return
	}

	// Close the reader and confirm that it finished.
	// Sets a trailer at the end of the HTTP response.
	if err := r.Close(); err == nil {
		// Temporary hack for writing trailers until Flynn uses Go 1.5.
		w.(http.Flusher).Flush()
		conn, buf, _ := w.(http.Hijacker).Hijack()
		buf.WriteString("0\r\n") // eof
		http.Header{backupChecksumTrailer: {hex.EncodeToString(hash.Sum(nil))}}.Write(buf)
		buf.WriteString("\r\n") // end of trailers
		buf.Flush()
		conn.Close()
	} else {
		h.Logger.Error("error returned from backup", "err", err)
	}
}

// handlePostStop handles request to POST /stop.
func (h *Handler) handlePostStop(w http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	if err := h.Peer.Stop(); err != nil {
		httphelper.Error(w, err)
		return
	}
	if err := h.Heartbeater.Close(); err != nil {
		httphelper.Error(w, err)
		return
	}
	w.WriteHeader(200)
}

const backupChecksumTrailer = "Flynn-Backup-Checksum"
<|endoftext|>


<commit_before>package handlers

import (
	"log"
	"net/http"
	"path/filepath"
	"strings"

	"github.com/Checksum/notify"
)

// FileWatcher recursively watches the served directory for changes
type FileWatcher struct {
	Handler    http.Handler
	Broker     *Broker
	Watcher    chan notify.EventInfo
	Extensions []string
}

// NewFileWatcher returns a new file watcher
func NewFileWatcher(dir, ext string, h http.Handler) *FileWatcher {
	b := NewBroker()
	c := make(chan notify.EventInfo)
	// Start the broker
	b.Start()
	// Start the listener
	err := notify.WatchWithFilter(dir+"/...", c, filterPath, notify.Create|notify.Write)

	if err != nil {
		log.Fatal(err)
	}

	fw := &FileWatcher{h, b, c, strings.Split(ext, ",")}

	go func() {
		for {
			select {
			case event := <-c:
				ext := filepath.Ext(event.Path())[1:]
				if fw.isValidExt(ext) {
					log.Println("File changed, reloading")
					b.Messages <- "reload:" + event.Path()
				}
			}
		}
	}()

	return fw

}

func (fw *FileWatcher) isValidExt(ext string) bool {
	for _, val := range fw.Extensions {
		if val == ext {
			return true
		}
	}
	return false
}

// This doesn't seem to be working!!
func filterPath(path string) bool {
	return true
}
<commit_msg>fix: fix extension validation<commit_after>package handlers

import (
	"log"
	"net/http"
	"path/filepath"
	"strings"

	"github.com/Checksum/notify"
)

// FileWatcher recursively watches the served directory for changes
type FileWatcher struct {
	Handler    http.Handler
	Broker     *Broker
	Watcher    chan notify.EventInfo
	Extensions []string
}

// NewFileWatcher returns a new file watcher
func NewFileWatcher(dir, ext string, h http.Handler) *FileWatcher {
	b := NewBroker()
	c := make(chan notify.EventInfo)
	// Start the broker
	b.Start()
	// Start the listener
	err := notify.WatchWithFilter(dir+"/...", c, filterPath, notify.Create|notify.Write)

	if err != nil {
		log.Fatal(err)
	}

	fw := &FileWatcher{h, b, c, strings.Split(ext, ",")}

	go func() {
		for {
			select {
			case event := <-c:
				ext := filepath.Ext(event.Path())
				if fw.isValidExt(ext) {
					log.Println("File changed, reloading")
					b.Messages <- "reload:" + event.Path()
				}
			}
		}
	}()

	return fw

}

func (fw *FileWatcher) isValidExt(ext string) bool {
	if ext != "" && len(ext) > 1 {
		ext = ext[1:]
		for _, val := range fw.Extensions {
			if val == ext {
				return true
			}
		}
	}
	return false
}

// This doesn't seem to be working!!
func filterPath(path string) bool {
	return true
}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"fmt"
	"github.com/qa-dev/jsonwire-grid/pool"
	log "github.com/sirupsen/logrus"
	"net/http"
)

// SessionInfo - Returns a session info (node address, status, etc)
type SessionInfo struct {
	Pool *pool.Pool
}

type sessionInfoResponse struct {
	NodeAddress string          `json:"node_address"`
	NodeType    pool.NodeType   `json:"node_type"`
	Status      pool.NodeStatus `json:"node_status"`
	SessionID   string          `json:"session_id"`
	Updated     int64           `json:"updated"`
	Registered  int64           `json:"registered"`
}

func (h *SessionInfo) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	sessionId := r.URL.Query().Get("sessionid")
	if sessionId == "" {
		http.Error(rw, fmt.Sprint("session id must be specified,"), http.StatusBadRequest)
		return
	}
	node, err := h.Pool.GetNodeBySessionID(sessionId)
	if err != nil {
		http.Error(rw, fmt.Sprint("trying to get a session data,", err), http.StatusInternalServerError)
		return
	}

	resp := sessionInfoResponse{
		node.Address,
		node.Type,
		node.Status,
		node.SessionID,
		node.Updated,
		node.Registered,
	}
	respJSON, err := json.Marshal(resp)
	if err != nil {
		http.Error(rw, fmt.Sprint("trying to encode a response,", err), http.StatusInternalServerError)
		return
	}

	_, err = rw.Write(respJSON)
	if err != nil {
		log.Error("session/info: write a response,", err)
	}
}
<commit_msg>Add API method '/grid/session/info'. Returns session info( address, port, etc.) by session id<commit_after>package handlers

import (
	"encoding/json"
	"fmt"
	"net/http"

	log "github.com/sirupsen/logrus"

	"github.com/qa-dev/jsonwire-grid/pool"
)

// SessionInfo - Returns a session info (node address, status, etc)
type SessionInfo struct {
	Pool *pool.Pool
}

type sessionInfoResponse struct {
	NodeAddress string          `json:"node_address"`
	NodeType    pool.NodeType   `json:"node_type"`
	Status      pool.NodeStatus `json:"node_status"`
	SessionID   string          `json:"session_id"`
	Updated     int64           `json:"updated"`
	Registered  int64           `json:"registered"`
}

func (h *SessionInfo) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	sessionId := r.URL.Query().Get("sessionid")
	if sessionId == "" {
		http.Error(rw, fmt.Sprint("session id must be specified,"), http.StatusBadRequest)
		return
	}
	node, err := h.Pool.GetNodeBySessionID(sessionId)
	if err != nil {
		http.Error(rw, fmt.Sprint("trying to get a session data,", err), http.StatusInternalServerError)
		return
	}

	resp := sessionInfoResponse{
		node.Address,
		node.Type,
		node.Status,
		node.SessionID,
		node.Updated,
		node.Registered,
	}
	respJSON, err := json.Marshal(resp)
	if err != nil {
		http.Error(rw, fmt.Sprint("trying to encode a response,", err), http.StatusInternalServerError)
		return
	}

	_, err = rw.Write(respJSON)
	if err != nil {
		log.Error("session/info: write a response,", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"os"

	"github.com/deiu/solidproxy"
)

func main() {
	config := solidproxy.NewServerConfig()

	if len(os.Getenv("SOLIDPROXY_VERBOSE")) == 0 {
		config.Verbose = false // default= true
	}
	if len(os.Getenv("SOLIDPROXY_PORT")) > 0 {
		config.Port = os.Getenv("SOLIDPROXY_PORT") // default= :3129
	}
	if len(os.Getenv("SOLIDPROXY_WEBID")) > 0 {
		config.WebID = os.Getenv("SOLIDPROXY_WEBID")
	}

	// Create a new server
	e := solidproxy.NewServer(config)
	// Start server
	println("Starting Solid-proxy", solidproxy.GetVersion())

	// set config values
	s := &http.Server{
		Addr:    ":" + config.Port,
		Handler: e,
	}
	// start server
	e.StartServer(s)
}
<commit_msg>fixed the package name<commit_after>package main

import (
	"net/http"
	"os"

	"github.com/solid/solidproxy"
)

func main() {
	config := solidproxy.NewServerConfig()

	if len(os.Getenv("SOLIDPROXY_VERBOSE")) == 0 {
		config.Verbose = false // default= true
	}
	if len(os.Getenv("SOLIDPROXY_PORT")) > 0 {
		config.Port = os.Getenv("SOLIDPROXY_PORT") // default= :3129
	}
	if len(os.Getenv("SOLIDPROXY_WEBID")) > 0 {
		config.WebID = os.Getenv("SOLIDPROXY_WEBID")
	}

	// Create a new server
	e := solidproxy.NewServer(config)
	// Start server
	println("Starting Solid-proxy", solidproxy.GetVersion())

	// set config values
	s := &http.Server{
		Addr:    ":" + config.Port,
		Handler: e,
	}
	// start server
	e.StartServer(s)
}
<|endoftext|>


<commit_before>package architecture

// https://github.com/opencontainers/image-spec/blob/v1.0.0-rc6/image-index.md#image-index-property-descriptions
// see "platform" (under "manifests")
type OCIPlatform struct {
	OS           string `json:"os"`
	Architecture string `json:"architecture"`
	Variant      string `json:"variant,omitempty"`

	//OSVersion  string   `json:"os.version,omitempty"`
	//OSFeatures []string `json:"os.features,omitempty"`
}

var SupportedArches = map[string]OCIPlatform{
	"amd64":   {OS: "linux", Architecture: "amd64"},
	"arm32v5": {OS: "linux", Architecture: "arm", Variant: "v5"},
	"arm32v6": {OS: "linux", Architecture: "arm", Variant: "v6"},
	"arm32v7": {OS: "linux", Architecture: "arm", Variant: "v7"},
	"arm64v8": {OS: "linux", Architecture: "arm64", Variant: "v8"},
	"i386":    {OS: "linux", Architecture: "386"},
	"ppc64le": {OS: "linux", Architecture: "ppc64le"},
	"s390x":   {OS: "linux", Architecture: "s390x"},

	"windows-amd64": {OS: "windows", Architecture: "amd64"},
}
<commit_msg>Add "mips64le" in SupportedArches<commit_after>package architecture

// https://github.com/opencontainers/image-spec/blob/v1.0.1/image-index.md#image-index-property-descriptions
// see "platform" (under "manifests")
type OCIPlatform struct {
	OS           string `json:"os"`
	Architecture string `json:"architecture"`
	Variant      string `json:"variant,omitempty"`

	//OSVersion  string   `json:"os.version,omitempty"`
	//OSFeatures []string `json:"os.features,omitempty"`
}

var SupportedArches = map[string]OCIPlatform{
	"amd64":    {OS: "linux", Architecture: "amd64"},
	"arm32v5":  {OS: "linux", Architecture: "arm", Variant: "v5"},
	"arm32v6":  {OS: "linux", Architecture: "arm", Variant: "v6"},
	"arm32v7":  {OS: "linux", Architecture: "arm", Variant: "v7"},
	"arm64v8":  {OS: "linux", Architecture: "arm64", Variant: "v8"},
	"i386":     {OS: "linux", Architecture: "386"},
	"mips64le": {OS: "linux", Architecture: "mips64le"},
	"ppc64le":  {OS: "linux", Architecture: "ppc64le"},
	"s390x":    {OS: "linux", Architecture: "s390x"},

	"windows-amd64": {OS: "windows", Architecture: "amd64"},
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/kylelemons/godebug/diff"
	"gopkg.in/yaml.v1"
)

// This file runs tests in the testdata directory, excluding those in testdata/single-file-tests

func TestCLISessions(t *testing.T) {
	godebug := compileGodebug(t)
	defer os.Remove(godebug)

	// Read the testdata directory
	fd, err := os.Open("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer fd.Close()
	names, err := fd.Readdirnames(-1)
	if err != nil {
		t.Fatal("Readdirnames:", err)
	}
	tests := make([]string, 0, len(names))
	for _, name := range names {
		if strings.HasSuffix(name, ".yaml") {
			tests = append(tests, name)
		}
	}

	// Run tests in parallel
	var wg sync.WaitGroup
	for _, test := range tests {
		for _, tt := range parseCases(t, filepath.Join("testdata", test)) {
			s := parseSessionFromBytes([]byte(tt.Transcript))
			for i := range tt.Invocations {
				wg.Add(1)
				go func(filename string, s *session, tt testCase, i int) {
					defer wg.Done()
					runTest(t, godebug, filename, tt, i, s)
				}(test, s, tt, i)
			}
		}
	}
	wg.Wait()
}

type testCase struct {
	Invocations []struct {
		Dir, Cmd string
	}
	Desc, Transcript string
	Creates          []string
	NonzeroExit      bool `yaml:"nonzero_exit"`
	Godebugwork      bool
}

func parseCases(t *testing.T, filename string) []testCase {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatal(err)
	}
	bb := bytes.Split(b, []byte("\n---\n"))
	testCases := make([]testCase, len(bb))
	for i := range bb {
		if err = yaml.Unmarshal(bb[i], &testCases[i]); err != nil {
			fmt.Println(string(bb[i]))
			t.Fatal(err)
		}
	}
	return testCases
}

func runTest(t *testing.T, godebug, filename string, tt testCase, i int, session *session) {
	var buf bytes.Buffer
	command, dir := tt.Invocations[i].Cmd, tt.Invocations[i].Dir
	cmd := exec.Command(godebug, strings.Split(command, " ")[1:]...)
	cmd.Dir = filepath.FromSlash("testdata/test-filesystem/" + dir)
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	cmd.Stdin = bytes.NewReader(session.input)
	setTestGopath(t, cmd)

	// Show multiple errors if they exist and format them nicely.
	var errs []string
	defer func() {
		if errs != nil {
			t.Errorf("File: %s\nDescription: %s\nWorking dir: %s\nCommand: %s\nFailures:\n\t%v",
				filename, tt.Desc, dir, command, strings.Join(errs, "\n\t"))
		}
	}()

	cmd.Env = append(cmd.Env, logFileEnvVar+"=true")
	err := cmd.Run()
	// Because we set `logFileEnvVar` above, godebug will print the
	// files it creates to stdout. Parse those lines and then pretend
	// they were not printed.
	createdFiles, output := recordCreatedFiles(buf.Bytes())

	switch err.(type) {
	case nil:
		if tt.NonzeroExit {
			errs = append(errs, "got exit code == 0, wanted a nonzero exit code.")
			return
		}
	case *exec.ExitError:
		if !tt.NonzeroExit {
			errs = append(errs, fmt.Sprintf("%q failed to run: %v\n%s", command, err, output))
			return
		}
	default:
		errs = append(errs, fmt.Sprintf("%q failed to run: %v\n%s", command, err, output))
		return
	}

	// Check that we created the files we expected and did not create
	// any files we did not expect.
	errs = append(errs, checkCreatedFiles(t, createdFiles, tt.Creates)...)

	if tt.Godebugwork {
		output, err = checkGodebugwork(t, session.fullSession, output)
		if err != nil {
			errs = append(errs, err.Error())
		}
	}

	got := interleaveCommands(session.input, output)
	if equivalent(got, session.fullSession) {
		return
	}
	errs = append(errs, fmt.Sprintf("golden transcript did not match actual transcript. Diff:\n\n%v", diff.Diff(string(session.fullSession), string(got))))
}

func checkGodebugwork(t *testing.T, transcript, output []byte) ([]byte, error) {
	if !bytes.HasPrefix(transcript, []byte("$TMP\n")) {
		return output, errors.New(`incorrect test: set "godebugwork: true" but did not prepend "$TMP\n" to the output`)
	}

	tmpDir := string(bytes.SplitN(output, newline, 2)[0])
	if !strings.HasPrefix(tmpDir, os.TempDir()) {
		return output, fmt.Errorf("got %q as first line of output, expected a temporary directory", tmpDir)
	}

	_, err := os.Stat(tmpDir)
	if os.IsNotExist(err) {
		return output, fmt.Errorf("godebug deleted the temporary directory %q when -godebugwork was passed", tmpDir)
	}

	if err != nil {
		return output, fmt.Errorf("failed to stat temporary directory %q:  %s", tmpDir, err)
	}

	output = append([]byte("$TMP\n"), output[len(tmpDir)+1:]...)
	if err = os.RemoveAll(tmpDir); err != nil {
		return output, fmt.Errorf("failed to remove temporary directory: %v", err)
	}

	return output, nil
}

func checkCreatedFiles(t *testing.T, g, w []string) (errs []string) {
	got, want := listToMap(g), listToMap(w)
	for f := range got {
		if !want[f] {
			errs = append(errs, "created a file we did not want: "+f)
		}
	}
	for f := range want {
		if !got[f] {
			errs = append(errs, "did not create a file we wanted: "+f)
		}
	}
	return errs
}

func recordCreatedFiles(b []byte) (files []string, rest []byte) {
	bb := bytes.Split(b, newline)

	for i := range bb {
		if bytes.HasPrefix(bb[i], []byte(logFilePrefix)) {
			files = append(files, string(bb[i][len(logFilePrefix):]))
		} else {
			rest = append(rest, bb[i]...)
			if i+1 < len(bb) {
				rest = append(rest, newline...)
			}
		}
	}
	return files, rest
}

func listToMap(list []string) map[string]bool {
	m := make(map[string]bool)
	for _, s := range list {
		m[s] = true
	}
	return m
}

// equivalent does a linewise comparison of a and b.
// Each line must be exactly equal or the want line must end in "//substr"
// and be a substring of the got line.
// Otherwise equivalent returns false.
func equivalent(got, want []byte) bool {
	var (
		gotLines  = bytes.Split(got, newline)
		wantLines = bytes.Split(want, newline)
		substr    = []byte("//substr")
		gg, ww    []byte
	)

	if len(gotLines) != len(wantLines) {
		return false
	}

	for i := range gotLines {
		gg, ww = gotLines[i], wantLines[i]
		if !(bytes.Equal(gg, ww) || bytes.HasSuffix(ww, substr) && bytes.Contains(gg, ww[:len(ww)-len(substr)])) {
			return false
		}
	}
	return true
}

func setTestGopath(t *testing.T, cmd *exec.Cmd) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	setGopath(cmd, filepath.Join(cwd, "testdata", "test-filesystem", "gopath"))
}
<commit_msg>yaml version bump<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"testing"

	"github.com/kylelemons/godebug/diff"
	"gopkg.in/yaml.v2"
)

// This file runs tests in the testdata directory, excluding those in testdata/single-file-tests

func TestCLISessions(t *testing.T) {
	godebug := compileGodebug(t)
	defer os.Remove(godebug)

	// Read the testdata directory
	fd, err := os.Open("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer fd.Close()
	names, err := fd.Readdirnames(-1)
	if err != nil {
		t.Fatal("Readdirnames:", err)
	}
	tests := make([]string, 0, len(names))
	for _, name := range names {
		if strings.HasSuffix(name, ".yaml") {
			tests = append(tests, name)
		}
	}

	// Run tests in parallel
	var wg sync.WaitGroup
	for _, test := range tests {
		for _, tt := range parseCases(t, filepath.Join("testdata", test)) {
			s := parseSessionFromBytes([]byte(tt.Transcript))
			for i := range tt.Invocations {
				wg.Add(1)
				go func(filename string, s *session, tt testCase, i int) {
					defer wg.Done()
					runTest(t, godebug, filename, tt, i, s)
				}(test, s, tt, i)
			}
		}
	}
	wg.Wait()
}

type testCase struct {
	Invocations []struct {
		Dir, Cmd string
	}
	Desc, Transcript string
	Creates          []string
	NonzeroExit      bool `yaml:"nonzero_exit"`
	Godebugwork      bool
}

func parseCases(t *testing.T, filename string) []testCase {
	b, err := ioutil.ReadFile(filename)
	if err != nil {
		t.Fatal(err)
	}
	bb := bytes.Split(b, []byte("\n---\n"))
	testCases := make([]testCase, len(bb))
	for i := range bb {
		if err = yaml.Unmarshal(bb[i], &testCases[i]); err != nil {
			fmt.Println(string(bb[i]))
			t.Fatal(err)
		}
	}
	return testCases
}

func runTest(t *testing.T, godebug, filename string, tt testCase, i int, session *session) {
	var buf bytes.Buffer
	command, dir := tt.Invocations[i].Cmd, tt.Invocations[i].Dir
	cmd := exec.Command(godebug, strings.Split(command, " ")[1:]...)
	cmd.Dir = filepath.FromSlash("testdata/test-filesystem/" + dir)
	cmd.Stdout = &buf
	cmd.Stderr = &buf
	cmd.Stdin = bytes.NewReader(session.input)
	setTestGopath(t, cmd)

	// Show multiple errors if they exist and format them nicely.
	var errs []string
	defer func() {
		if errs != nil {
			t.Errorf("File: %s\nDescription: %s\nWorking dir: %s\nCommand: %s\nFailures:\n\t%v",
				filename, tt.Desc, dir, command, strings.Join(errs, "\n\t"))
		}
	}()

	cmd.Env = append(cmd.Env, logFileEnvVar+"=true")
	err := cmd.Run()
	// Because we set `logFileEnvVar` above, godebug will print the
	// files it creates to stdout. Parse those lines and then pretend
	// they were not printed.
	createdFiles, output := recordCreatedFiles(buf.Bytes())

	switch err.(type) {
	case nil:
		if tt.NonzeroExit {
			errs = append(errs, "got exit code == 0, wanted a nonzero exit code.")
			return
		}
	case *exec.ExitError:
		if !tt.NonzeroExit {
			errs = append(errs, fmt.Sprintf("%q failed to run: %v\n%s", command, err, output))
			return
		}
	default:
		errs = append(errs, fmt.Sprintf("%q failed to run: %v\n%s", command, err, output))
		return
	}

	// Check that we created the files we expected and did not create
	// any files we did not expect.
	errs = append(errs, checkCreatedFiles(t, createdFiles, tt.Creates)...)

	if tt.Godebugwork {
		output, err = checkGodebugwork(t, session.fullSession, output)
		if err != nil {
			errs = append(errs, err.Error())
		}
	}

	got := interleaveCommands(session.input, output)
	if equivalent(got, session.fullSession) {
		return
	}
	errs = append(errs, fmt.Sprintf("golden transcript did not match actual transcript. Diff:\n\n%v", diff.Diff(string(session.fullSession), string(got))))
}

func checkGodebugwork(t *testing.T, transcript, output []byte) ([]byte, error) {
	if !bytes.HasPrefix(transcript, []byte("$TMP\n")) {
		return output, errors.New(`incorrect test: set "godebugwork: true" but did not prepend "$TMP\n" to the output`)
	}

	tmpDir := string(bytes.SplitN(output, newline, 2)[0])
	if !strings.HasPrefix(tmpDir, os.TempDir()) {
		return output, fmt.Errorf("got %q as first line of output, expected a temporary directory", tmpDir)
	}

	_, err := os.Stat(tmpDir)
	if os.IsNotExist(err) {
		return output, fmt.Errorf("godebug deleted the temporary directory %q when -godebugwork was passed", tmpDir)
	}

	if err != nil {
		return output, fmt.Errorf("failed to stat temporary directory %q:  %s", tmpDir, err)
	}

	output = append([]byte("$TMP\n"), output[len(tmpDir)+1:]...)
	if err = os.RemoveAll(tmpDir); err != nil {
		return output, fmt.Errorf("failed to remove temporary directory: %v", err)
	}

	return output, nil
}

func checkCreatedFiles(t *testing.T, g, w []string) (errs []string) {
	got, want := listToMap(g), listToMap(w)
	for f := range got {
		if !want[f] {
			errs = append(errs, "created a file we did not want: "+f)
		}
	}
	for f := range want {
		if !got[f] {
			errs = append(errs, "did not create a file we wanted: "+f)
		}
	}
	return errs
}

func recordCreatedFiles(b []byte) (files []string, rest []byte) {
	bb := bytes.Split(b, newline)

	for i := range bb {
		if bytes.HasPrefix(bb[i], []byte(logFilePrefix)) {
			files = append(files, string(bb[i][len(logFilePrefix):]))
		} else {
			rest = append(rest, bb[i]...)
			if i+1 < len(bb) {
				rest = append(rest, newline...)
			}
		}
	}
	return files, rest
}

func listToMap(list []string) map[string]bool {
	m := make(map[string]bool)
	for _, s := range list {
		m[s] = true
	}
	return m
}

// equivalent does a linewise comparison of a and b.
// Each line must be exactly equal or the want line must end in "//substr"
// and be a substring of the got line.
// Otherwise equivalent returns false.
func equivalent(got, want []byte) bool {
	var (
		gotLines  = bytes.Split(got, newline)
		wantLines = bytes.Split(want, newline)
		substr    = []byte("//substr")
		gg, ww    []byte
	)

	if len(gotLines) != len(wantLines) {
		return false
	}

	for i := range gotLines {
		gg, ww = gotLines[i], wantLines[i]
		if !(bytes.Equal(gg, ww) || bytes.HasSuffix(ww, substr) && bytes.Contains(gg, ww[:len(ww)-len(substr)])) {
			return false
		}
	}
	return true
}

func setTestGopath(t *testing.T, cmd *exec.Cmd) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	setGopath(cmd, filepath.Join(cwd, "testdata", "test-filesystem", "gopath"))
}
<|endoftext|>


<commit_before>package cdn

import (
	"context"
	"time"

	"github.com/rockbears/log"

	"github.com/ovh/cds/engine/cdn/item"
	"github.com/ovh/cds/engine/cdn/storage"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/telemetry"
)

const (
	ItemLogGC = 24 * 3600
)

func (s *Service) itemPurge(ctx context.Context) {
	tickPurge := time.NewTicker(1 * time.Minute)
	defer tickPurge.Stop()
	for {
		select {
		case <-ctx.Done():
			if ctx.Err() != nil {
				log.Error(ctx, "cdn:ItemPurge: %v", ctx.Err())
			}
			return
		case <-tickPurge.C:
			if err := s.cleanItemToDelete(ctx); err != nil {
				ctx = sdk.ContextWithStacktrace(ctx, err)
				log.Error(ctx, err.Error())
			}
		}
	}
}

// ItemsGC clean long incoming item + delete item from buffer when synchronized everywhere
func (s *Service) itemsGC(ctx context.Context) {
	tickGC := time.NewTicker(1 * time.Minute)
	defer tickGC.Stop()
	for {
		select {
		case <-ctx.Done():
			if ctx.Err() != nil {
				log.Error(ctx, "cdn:CompleteWaitingItems: %v", ctx.Err())
			}
			return
		case <-tickGC.C:
			if err := s.cleanBuffer(ctx); err != nil {
				ctx = sdk.ContextWithStacktrace(ctx, err)
				log.Error(ctx, err.Error())
			}
			if err := s.cleanWaitingItem(ctx, ItemLogGC); err != nil {
				ctx = sdk.ContextWithStacktrace(ctx, err)
				log.Error(ctx, err.Error())
			}
		}
	}
}

func (s *Service) markUnitItemToDeleteByItemID(ctx context.Context, itemID string) (int, error) {
	db := s.mustDBWithCtx(ctx)
	itemUnitIDs, err := storage.LoadAllItemUnitsIDsByItemID(db, itemID)
	if err != nil {
		return 0, err
	}
	if len(itemUnitIDs) == 0 {
		return 0, nil
	}

	tx, err := db.Begin()
	if err != nil {
		return 0, sdk.WithStack(err)
	}

	defer tx.Rollback() // nolint

	n, err := storage.MarkItemUnitToDelete(tx, itemUnitIDs)
	if err != nil {
		return 0, err
	}

	return n, sdk.WithStack(tx.Commit())
}

func (s *Service) cleanItemToDelete(ctx context.Context) error {
	ids, err := item.LoadIDsToDelete(s.mustDBWithCtx(ctx), 100)
	if err != nil {
		return err
	}

	if len(ids) > 0 {
		log.Info(ctx, "cdn:purge:item: %d items to delete", len(ids))
	}

	for _, id := range ids {
		nbUnitItemToDelete, err := s.markUnitItemToDeleteByItemID(ctx, id)
		if err != nil {
			log.Error(ctx, "unable to mark unit item %q to delete: %v", id, err)
			continue
		}

		// If and only If there is not more unit item to mark as delete,
		// let's delete the item in database
		if nbUnitItemToDelete == 0 {
			nbItemUnits, err := storage.CountItemUnitsToDeleteByItemID(s.mustDBWithCtx(ctx), id)
			if err != nil {
				log.Error(ctx, "unable to count unit item %q to delete: %v", id, err)
				continue
			}

			if nbItemUnits > 0 {
				log.Debug(ctx, "cdn:purge:item: %d unit items to delete for item %s", nbItemUnits, id)
			} else {
				if err := s.LogCache.Remove([]string{id}); err != nil {
					return err
				}
				if err := item.DeleteByID(s.mustDBWithCtx(ctx), id); err != nil {
					return err
				}
				for _, sto := range s.Units.Storages {
					s.Units.RemoveFromRedisSyncQueue(ctx, sto, id)
				}

				log.Debug(ctx, "cdn:purge:item: %s item deleted", id)
			}
			continue
		}

		log.Debug(ctx, "cdn:purge:item: %d unit items to delete for item %s", nbUnitItemToDelete, id)
	}
	return nil
}

func (s *Service) cleanBuffer(ctx context.Context) error {
	storageCount := int64(len(s.Units.Storages) + 1)
	for _, bu := range s.Units.Buffers {
		itemIDs, err := storage.LoadAllSynchronizedItemIDs(s.mustDBWithCtx(ctx), bu.ID(), storageCount)
		if err != nil {
			return err
		}
		log.Debug(ctx, "item to remove from buffer: %d", len(itemIDs))
		if len(itemIDs) == 0 {
			return nil
		}

		itemUnitsIDs, err := storage.LoadAllItemUnitsIDsByItemIDsAndUnitID(s.mustDBWithCtx(ctx), bu.ID(), itemIDs)
		if err != nil {
			return err
		}

		tx, err := s.mustDBWithCtx(ctx).Begin()
		if err != nil {
			return sdk.WrapError(err, "unable to start transaction")
		}

		if _, err := storage.MarkItemUnitToDelete(tx, itemUnitsIDs); err != nil {
			_ = tx.Rollback()
			return err
		}

		if err := tx.Commit(); err != nil {
			_ = tx.Rollback()
			return sdk.WithStack(err)
		}
	}
	return nil
}

func (s *Service) cleanWaitingItem(ctx context.Context, duration int) error {
	itemUnits, err := storage.LoadOldItemUnitByItemStatusAndDuration(ctx, s.Mapper, s.mustDBWithCtx(ctx), sdk.CDNStatusItemIncoming, duration)
	if err != nil {
		return err
	}
	for _, itemUnit := range itemUnits {
		ctx = context.WithValue(ctx, storage.FieldAPIRef, itemUnit.Item.APIRef)
		log.Info(ctx, "cleanWaitingItem> cleaning item %s", itemUnit.ItemID)

		tx, err := s.mustDBWithCtx(ctx).Begin()
		if err != nil {
			return sdk.WrapError(err, "unable to start transaction")
		}
		if err := s.completeItem(ctx, tx, itemUnit); err != nil {
			_ = tx.Rollback()
			return err
		}
		if err := tx.Commit(); err != nil {
			_ = tx.Rollback()
			return err
		}
		s.Units.PushInSyncQueue(ctx, itemUnit.ItemID, itemUnit.Item.Created)
		telemetry.Record(ctx, s.Metrics.itemCompletedByGCCount, 1)
	}
	return nil
}
<commit_msg>fix(cdn): add some log on gc (#5685)<commit_after>package cdn

import (
	"context"
	"time"

	"github.com/rockbears/log"

	"github.com/ovh/cds/engine/cdn/item"
	"github.com/ovh/cds/engine/cdn/storage"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/telemetry"
)

const (
	ItemLogGC = 24 * 3600
)

func (s *Service) itemPurge(ctx context.Context) {
	tickPurge := time.NewTicker(1 * time.Minute)
	defer tickPurge.Stop()
	for {
		select {
		case <-ctx.Done():
			if ctx.Err() != nil {
				log.Error(ctx, "cdn:ItemPurge: %v", ctx.Err())
			}
			return
		case <-tickPurge.C:
			if err := s.cleanItemToDelete(ctx); err != nil {
				ctx = sdk.ContextWithStacktrace(ctx, err)
				log.Error(ctx, "cdn:ItemPurge: error on cleanItemToDelete: %v", err)
			}
		}
	}
}

// ItemsGC clean long incoming item + delete item from buffer when synchronized everywhere
func (s *Service) itemsGC(ctx context.Context) {
	tickGC := time.NewTicker(1 * time.Minute)
	defer tickGC.Stop()
	for {
		select {
		case <-ctx.Done():
			if ctx.Err() != nil {
				log.Error(ctx, "cdn:CompleteWaitingItems: %v", ctx.Err())
			}
			return
		case <-tickGC.C:
			if err := s.cleanBuffer(ctx); err != nil {
				ctx = sdk.ContextWithStacktrace(ctx, err)
				log.Error(ctx, "cdn:CompleteWaitingItems: cleanBuffer err: %v", err)
			}
			if err := s.cleanWaitingItem(ctx, ItemLogGC); err != nil {
				ctx = sdk.ContextWithStacktrace(ctx, err)
				log.Error(ctx, "cdn:CompleteWaitingItems: ContextWithStacktrace err: %v", err)
			}
		}
	}
}

func (s *Service) markUnitItemToDeleteByItemID(ctx context.Context, itemID string) (int, error) {
	db := s.mustDBWithCtx(ctx)
	itemUnitIDs, err := storage.LoadAllItemUnitsIDsByItemID(db, itemID)
	if err != nil {
		return 0, err
	}
	if len(itemUnitIDs) == 0 {
		return 0, nil
	}

	tx, err := db.Begin()
	if err != nil {
		return 0, sdk.WithStack(err)
	}

	defer tx.Rollback() // nolint

	n, err := storage.MarkItemUnitToDelete(tx, itemUnitIDs)
	if err != nil {
		return 0, err
	}

	return n, sdk.WithStack(tx.Commit())
}

func (s *Service) cleanItemToDelete(ctx context.Context) error {
	ids, err := item.LoadIDsToDelete(s.mustDBWithCtx(ctx), 100)
	if err != nil {
		return err
	}

	if len(ids) > 0 {
		log.Info(ctx, "cdn:purge:item: %d items to delete", len(ids))
	}

	for _, id := range ids {
		nbUnitItemToDelete, err := s.markUnitItemToDeleteByItemID(ctx, id)
		if err != nil {
			log.Error(ctx, "cdn:purge:item: unable to mark unit item %q to delete: %v", id, err)
			continue
		}

		log.Debug(ctx, "cdn:purge:item: %d unit items to delete for item %q", nbUnitItemToDelete, id)

		// If and only If there is not more unit item to mark as delete,
		// let's delete the item in database
		if nbUnitItemToDelete == 0 {
			nbItemUnits, err := storage.CountItemUnitsToDeleteByItemID(s.mustDBWithCtx(ctx), id)
			if err != nil {
				log.Error(ctx, "cdn:purge:item: unable to count unit item %q to delete: %v", id, err)
				continue
			}

			if nbItemUnits > 0 {
				log.Debug(ctx, "cdn:purge:item: %d unit items to delete for item %q", nbItemUnits, id)
			} else {
				if err := s.LogCache.Remove([]string{id}); err != nil {
					return sdk.WrapError(err, "cdn:purge:item: unable to remove from logCache for item %q")
				}
				if err := item.DeleteByID(s.mustDBWithCtx(ctx), id); err != nil {
					return sdk.WrapError(err, "cdn:purge:item: unable to delete from item with id %q")
				}
				for _, sto := range s.Units.Storages {
					s.Units.RemoveFromRedisSyncQueue(ctx, sto, id)
				}

				log.Debug(ctx, "cdn:purge:item: %s item deleted", id)
			}
		}
	}
	return nil
}

func (s *Service) cleanBuffer(ctx context.Context) error {
	storageCount := int64(len(s.Units.Storages) + 1)
	for _, bu := range s.Units.Buffers {
		itemIDs, err := storage.LoadAllSynchronizedItemIDs(s.mustDBWithCtx(ctx), bu.ID(), storageCount)
		if err != nil {
			return err
		}
		log.Debug(ctx, "item to remove from buffer: %d", len(itemIDs))
		if len(itemIDs) == 0 {
			return nil
		}

		itemUnitsIDs, err := storage.LoadAllItemUnitsIDsByItemIDsAndUnitID(s.mustDBWithCtx(ctx), bu.ID(), itemIDs)
		if err != nil {
			return err
		}

		tx, err := s.mustDBWithCtx(ctx).Begin()
		if err != nil {
			return sdk.WrapError(err, "unable to start transaction")
		}

		if _, err := storage.MarkItemUnitToDelete(tx, itemUnitsIDs); err != nil {
			_ = tx.Rollback()
			return err
		}

		if err := tx.Commit(); err != nil {
			_ = tx.Rollback()
			return sdk.WithStack(err)
		}
	}
	return nil
}

func (s *Service) cleanWaitingItem(ctx context.Context, duration int) error {
	itemUnits, err := storage.LoadOldItemUnitByItemStatusAndDuration(ctx, s.Mapper, s.mustDBWithCtx(ctx), sdk.CDNStatusItemIncoming, duration)
	if err != nil {
		return err
	}
	for _, itemUnit := range itemUnits {
		ctx = context.WithValue(ctx, storage.FieldAPIRef, itemUnit.Item.APIRef)
		log.Info(ctx, "cleanWaitingItem> cleaning item %s", itemUnit.ItemID)

		tx, err := s.mustDBWithCtx(ctx).Begin()
		if err != nil {
			return sdk.WrapError(err, "unable to start transaction")
		}
		if err := s.completeItem(ctx, tx, itemUnit); err != nil {
			_ = tx.Rollback()
			return err
		}
		if err := tx.Commit(); err != nil {
			_ = tx.Rollback()
			return err
		}
		s.Units.PushInSyncQueue(ctx, itemUnit.ItemID, itemUnit.Item.Created)
		telemetry.Record(ctx, s.Metrics.itemCompletedByGCCount, 1)
	}
	return nil
}
<|endoftext|>


<commit_before>package dataframe

import (
	"bytes"
	"errors"
	"strconv"
)

// Errors
var (
	ErrInvalidTypesLen       = errors.New("the length of types does not match with the one of item names")
	ErrInvalidType           = errors.New("invalid type")
	ErrDuplicatedItemName    = errors.New("duplicated itemName")
	ErrNoData                = errors.New("no data")
	ErrInvalidDataColsNum    = errors.New("invalid number of data columns")
	ErrItemNameAlreadyExists = errors.New("itemName already exists")
	ErrItemNameNotExist      = errors.New("itemName does not exist")
)

// DataFrame represents a data frame.
type DataFrame struct {
	bd         *baseData
	fromRowIdx int // inclusive
	toRowIdx   int // exclusive
}

// RowNum returns the number of rows.
func (df *DataFrame) RowNum() int {
	return df.toRowIdx - df.fromRowIdx
}

// ColNum returns the number of columns.
func (df *DataFrame) ColNum() int {
	return len(df.bd.itemNames)
}

// Head creates a new data frame which has top n rows of
// the original data frame.
func (df *DataFrame) Head(n int) *DataFrame {
	return &DataFrame{df.bd, df.fromRowIdx, min(df.fromRowIdx+n, df.toRowIdx)}
}

// Tail creates a new data frame which has last n rows of
// the original data frame.
func (df *DataFrame) Tail(n int) *DataFrame {
	return &DataFrame{df.bd, max(df.toRowIdx-n, df.fromRowIdx), df.toRowIdx}
}

// String returns the string expression of the data frame.
func (df *DataFrame) String() string {
	bf := bytes.NewBufferString("")

	for i, itemName := range df.bd.itemNames {
		if i > 0 {
			bf.WriteRune(' ')
		}

		bf.WriteString(itemName)
	}

	bf.WriteRune('\n')

	for i, n := 0, min(maxPrintRows, (df.toRowIdx-df.fromRowIdx)); i < n; i++ {
		if i > 0 {
			bf.WriteRune('\n')
		}

		for j, itemName := range df.bd.itemNames {
			if j > 0 {
				bf.WriteRune(' ')
			}

			t := df.bd.types[itemName]

			if t == String {
				bf.WriteString(df.bd.stringCols[itemName][i+df.fromRowIdx])
			} else {
				bf.WriteString(strconv.FormatFloat(df.bd.float64Cols[itemName][i+df.fromRowIdx], 'f', 8, 64))
			}
		}
	}

	return bf.String()
}

// AppendFloat64ColFromStringCol creates a float64 column from a string column and
// appends it to the data frame.
func (df *DataFrame) AppendFloat64ColFromStringCol(itemName, srcItemName string, convert func(string) (float64, error)) error {
	if _, exist := df.bd.stringCols[itemName]; exist {
		return ErrItemNameAlreadyExists
	}

	if _, exist := df.bd.float64Cols[itemName]; exist {
		return ErrItemNameAlreadyExists
	}

	stringCol, exist := df.bd.stringCols[srcItemName]
	if !exist {
		return ErrItemNameNotExist
	}

	n := len(stringCol)

	float64Col := make([]float64, n)

	ch := make(chan error, numConcurrency)

	d := divUp(n, numConcurrency)

	for i := 0; i < numConcurrency; i++ {
		from := d * i
		to := min(d*(i+1), n)

		go setFloat64FromString(float64Col, stringCol, from, to, convert, ch)
	}

	errs := make([]error, 0, numConcurrency)

	for i := 0; i < numConcurrency; i++ {
		err := <-ch
		if err != nil {
			errs = append(errs, err)
		}
	}

	if len(errs) > 0 {
		return &MultiError{",", errs}
	}

	df.bd.itemNames = append(df.bd.itemNames, itemName)
	df.bd.types[itemName] = Float64
	df.bd.float64Cols[itemName] = float64Col

	return nil
}

// setFloat64FromString creates a float64 data from a string data and
// appends it to the slice.
func setFloat64FromString(float64Col []float64, stringCol []string, from, to int, convert func(string) (float64, error), ch chan<- error) {
	for i := from; i < to; i++ {
		f, err := convert(stringCol[i])
		if err != nil {
			ch <- err
			return
		}

		float64Col[i] = f
	}

	ch <- nil
}

// New creates and returns a data frame.
func New(data [][]string, config Config) (*DataFrame, error) {
	bd, err := newBaseData(data, config)
	if err != nil {
		return nil, err
	}

	return &DataFrame{bd, 0, bd.rowNum()}, nil
}
<commit_msg>Update dataframe/dataframe.go<commit_after>package dataframe

import (
	"bytes"
	"errors"
	"strconv"
	"sync"
)

// Errors
var (
	ErrInvalidTypesLen       = errors.New("the length of types does not match with the one of item names")
	ErrInvalidType           = errors.New("invalid type")
	ErrDuplicatedItemName    = errors.New("duplicated itemName")
	ErrNoData                = errors.New("no data")
	ErrInvalidDataColsNum    = errors.New("invalid number of data columns")
	ErrItemNameAlreadyExists = errors.New("itemName already exists")
	ErrItemNameNotExist      = errors.New("itemName does not exist")
)

// DataFrame represents a data frame.
type DataFrame struct {
	bd         *baseData
	fromRowIdx int // inclusive
	toRowIdx   int // exclusive
}

// RowNum returns the number of rows.
func (df *DataFrame) RowNum() int {
	return df.toRowIdx - df.fromRowIdx
}

// ColNum returns the number of columns.
func (df *DataFrame) ColNum() int {
	return len(df.bd.itemNames)
}

// Head creates a new data frame which has top n rows of
// the original data frame.
func (df *DataFrame) Head(n int) *DataFrame {
	return &DataFrame{df.bd, df.fromRowIdx, min(df.fromRowIdx+n, df.toRowIdx)}
}

// Tail creates a new data frame which has last n rows of
// the original data frame.
func (df *DataFrame) Tail(n int) *DataFrame {
	return &DataFrame{df.bd, max(df.toRowIdx-n, df.fromRowIdx), df.toRowIdx}
}

// String returns the string expression of the data frame.
func (df *DataFrame) String() string {
	bf := bytes.NewBufferString("")

	for i, itemName := range df.bd.itemNames {
		if i > 0 {
			bf.WriteRune(' ')
		}

		bf.WriteString(itemName)
	}

	bf.WriteRune('\n')

	for i, n := 0, min(maxPrintRows, (df.toRowIdx-df.fromRowIdx)); i < n; i++ {
		if i > 0 {
			bf.WriteRune('\n')
		}

		for j, itemName := range df.bd.itemNames {
			if j > 0 {
				bf.WriteRune(' ')
			}

			t := df.bd.types[itemName]

			if t == String {
				bf.WriteString(df.bd.stringCols[itemName][i+df.fromRowIdx])
			} else {
				bf.WriteString(strconv.FormatFloat(df.bd.float64Cols[itemName][i+df.fromRowIdx], 'f', 8, 64))
			}
		}
	}

	return bf.String()
}

// AppendFloat64ColFromStringCol creates a float64 column from a string column and
// appends it to the data frame.
func (df *DataFrame) AppendFloat64ColFromStringCol(itemName, srcItemName string, convert func(string) (float64, error)) error {
	if _, exist := df.bd.stringCols[itemName]; exist {
		return ErrItemNameAlreadyExists
	}

	if _, exist := df.bd.float64Cols[itemName]; exist {
		return ErrItemNameAlreadyExists
	}

	stringCol, exist := df.bd.stringCols[srcItemName]
	if !exist {
		return ErrItemNameNotExist
	}

	n := len(stringCol)

	float64Col := make([]float64, n)

	ch := make(chan error, numConcurrency)

	d := divUp(n, numConcurrency)

	for i := 0; i < numConcurrency; i++ {
		from := d * i
		to := min(d*(i+1), n)

		go setFloat64FromString(float64Col, stringCol, from, to, convert, ch)
	}

	errs := make([]error, 0, numConcurrency)

	for i := 0; i < numConcurrency; i++ {
		err := <-ch
		if err != nil {
			errs = append(errs, err)
		}
	}

	if len(errs) > 0 {
		return &MultiError{",", errs}
	}

	df.bd.itemNames = append(df.bd.itemNames, itemName)
	df.bd.types[itemName] = Float64
	df.bd.float64Cols[itemName] = float64Col

	return nil
}

// Float64Values creates and returns float64 2d slice.
func (df *DataFrame) Float64Values(itemNames []string) ([][]float64, error) {
	n := df.RowNum()

	v := make([][]float64, n)

	cn := len(itemNames)

	float64Cols := make([][]float64, cn)

	for i, itemName := range itemNames {
		float64Col, exist := df.bd.float64Cols[itemName]
		if !exist {
			return nil, ErrItemNameNotExist
		}

		float64Cols[i] = float64Col
	}

	wg := new(sync.WaitGroup)
	wg.Add(numConcurrency)

	d := divUp(n, numConcurrency)

	for i := 0; i < numConcurrency; i++ {
		from := df.fromRowIdx + d*i
		to := df.fromRowIdx + min(d*(i+1), n)

		go setFloat64Values(v, float64Cols, cn, from, to, wg)
	}

	wg.Wait()

	return v, nil
}

// setFloat64Values sets float64 values to v.
func setFloat64Values(v, float64Cols [][]float64, cn int, from, to int, wg *sync.WaitGroup) {
	defer wg.Done()

	for i := from; i < to; i++ {
		v[i] = make([]float64, cn)

		for j := 0; j < cn; j++ {
			v[i][j] = float64Cols[j][i]
		}
	}
}

// setFloat64FromString creates a float64 data from a string data and
// appends it to the slice.
func setFloat64FromString(float64Col []float64, stringCol []string, from, to int, convert func(string) (float64, error), ch chan<- error) {
	for i := from; i < to; i++ {
		f, err := convert(stringCol[i])
		if err != nil {
			ch <- err
			return
		}

		float64Col[i] = f
	}

	ch <- nil
}

// New creates and returns a data frame.
func New(data [][]string, config Config) (*DataFrame, error) {
	bd, err := newBaseData(data, config)
	if err != nil {
		return nil, err
	}

	return &DataFrame{bd, 0, bd.rowNum()}, nil
}
<|endoftext|>


<commit_before>package state

import (
	"fmt"
	"launchpad.net/juju-core/state/presence"
	"path"
	"strconv"
	"strings"
	"time"
)

const providerMachineId = "provider-machine-id"

type NoInstanceIdError struct {
	machineId int
}

func (e NoInstanceIdError) Error() string {
	return fmt.Sprintf("instance id for machine %d is not set", e.machineId)
}

// Machine represents the state of a machine.
type Machine struct {
	st  *State
	key string
}

// Id returns the machine id.
func (m *Machine) Id() int {
	return keySeq(m.key)
}

// AgentAlive returns whether the respective remote agent is alive.
func (m *Machine) AgentAlive() (bool, error) {
	return presence.Alive(m.st.zk, m.zkAgentPath())
}

// WaitAgentAlive blocks until the respective agent is alive.
func (m *Machine) WaitAgentAlive(timeout time.Duration) error {
	err := presence.WaitAlive(m.st.zk, m.zkAgentPath(), timeout)
	if err != nil {
		return fmt.Errorf("waiting for agent of machine %s: %v", m, err)
	}
	return nil
}

// SetAgentAlive signals that the agent for machine m is alive
// by starting a pinger on its presence node. It returns the
// started pinger.
func (m *Machine) SetAgentAlive() (*presence.Pinger, error) {
	return presence.StartPinger(m.st.zk, m.zkAgentPath(), agentPingerPeriod)
}

// InstanceId returns the provider specific machine id for this machine.
// If the id is not set, or it's value is "" and error of type NoInstanceIdError
// will be returned.
func (m *Machine) InstanceId() (string, error) {
	config, err := readConfigNode(m.st.zk, m.zkPath())
	if err != nil {
		return "", fmt.Errorf("can't get instance id of machine %s: %v", m, err)
	}
	v, ok := config.Get(providerMachineId)
	if !ok {
		return "", NoInstanceIdError{m.Id()}
	}
	if id, ok := v.(string); ok {
		if id == "" {
			return "", NoInstanceIdError{m.Id()}
		}
		return id, nil
	}
	return "", fmt.Errorf("invalid internal machine id type %T for machine %s", v, m)
}

// Units returns all the units that have been assigned
// to the machine.
func (m *Machine) Units() (units []*Unit, err error) {
	defer errorContextf(&err, "can't get all assigned units of machine %s", m)
	topology, err := readTopology(m.st.zk)
	if err != nil {
		return nil, err
	}
	keys := topology.UnitsForMachine(m.key)
	units = make([]*Unit, len(keys))
	for i, key := range keys {
		units[i], err = m.st.unitFromKey(topology, key)
		if err != nil {
			return nil, fmt.Errorf("inconsistent topology: %v", err)
		}
	}
	return units, nil
}

func (m *Machine) WatchUnits() *MachineUnitsWatcher {
	return newMachineUnitsWatcher(m)
}

// SetInstanceId sets the provider specific machine id for this machine.
func (m *Machine) SetInstanceId(id string) (err error) {
	defer errorContextf(&err, "can't set instance id of machine %s to %q", m, id)
	config, err := readConfigNode(m.st.zk, m.zkPath())
	if err != nil {
		return err
	}
	config.Set(providerMachineId, id)
	_, err = config.Write()
	return err
}

// String returns a unique description of this machine
func (m *Machine) String() string {
	return strconv.Itoa(m.Id())
}

// zkPath returns the ZooKeeper base path for the machine.
func (m *Machine) zkPath() string {
	return path.Join(zkMachinesPath, m.key)
}

// zkAgentPath returns the ZooKeeper path for the machine agent.
func (m *Machine) zkAgentPath() string {
	return path.Join(m.zkPath(), "agent")
}

// keySeq returns the sequence number part of
// the the given machine or unit key.
func keySeq(key string) (id int) {
	if key == "" {
		panic("keySeq: empty key")
	}
	i := strings.LastIndex(key, "-")
	var id64 int64
	var err error
	if i >= 0 {
		id64, err = strconv.ParseInt(key[i+1:], 10, 32)
	}
	if i < 0 || err != nil {
		panic("keySeq: invalid key: " + key)
	}
	return int(id64)
}

// machineKey returns the machine key corresponding to machineId.
func machineKey(machineId int) string {
	return fmt.Sprintf("machine-%010d", machineId)
}
<commit_msg>final tweak<commit_after>package state

import (
	"fmt"
	"launchpad.net/juju-core/state/presence"
	"path"
	"strconv"
	"strings"
	"time"
)

const providerMachineId = "provider-machine-id"

type NoInstanceIdError struct {
	machineId int
}

func (e NoInstanceIdError) Error() string {
	return fmt.Sprintf("instance id for machine %d is not set", e.machineId)
}

// Machine represents the state of a machine.
type Machine struct {
	st  *State
	key string
}

// Id returns the machine id.
func (m *Machine) Id() int {
	return keySeq(m.key)
}

// AgentAlive returns whether the respective remote agent is alive.
func (m *Machine) AgentAlive() (bool, error) {
	return presence.Alive(m.st.zk, m.zkAgentPath())
}

// WaitAgentAlive blocks until the respective agent is alive.
func (m *Machine) WaitAgentAlive(timeout time.Duration) error {
	err := presence.WaitAlive(m.st.zk, m.zkAgentPath(), timeout)
	if err != nil {
		return fmt.Errorf("waiting for agent of machine %s: %v", m, err)
	}
	return nil
}

// SetAgentAlive signals that the agent for machine m is alive
// by starting a pinger on its presence node. It returns the
// started pinger.
func (m *Machine) SetAgentAlive() (*presence.Pinger, error) {
	return presence.StartPinger(m.st.zk, m.zkAgentPath(), agentPingerPeriod)
}

// InstanceId returns the provider specific machine id for this machine.
// If the id is not set, or its value is "" and error of type NoInstanceIdError
// will be returned.
func (m *Machine) InstanceId() (string, error) {
	config, err := readConfigNode(m.st.zk, m.zkPath())
	if err != nil {
		return "", fmt.Errorf("can't get instance id of machine %s: %v", m, err)
	}
	v, ok := config.Get(providerMachineId)
	if !ok {
		return "", NoInstanceIdError{m.Id()}
	}
	if id, ok := v.(string); ok {
		if id == "" {
			return "", NoInstanceIdError{m.Id()}
		}
		return id, nil
	}
	return "", fmt.Errorf("invalid internal machine id type %T for machine %s", v, m)
}

// Units returns all the units that have been assigned
// to the machine.
func (m *Machine) Units() (units []*Unit, err error) {
	defer errorContextf(&err, "can't get all assigned units of machine %s", m)
	topology, err := readTopology(m.st.zk)
	if err != nil {
		return nil, err
	}
	keys := topology.UnitsForMachine(m.key)
	units = make([]*Unit, len(keys))
	for i, key := range keys {
		units[i], err = m.st.unitFromKey(topology, key)
		if err != nil {
			return nil, fmt.Errorf("inconsistent topology: %v", err)
		}
	}
	return units, nil
}

func (m *Machine) WatchUnits() *MachineUnitsWatcher {
	return newMachineUnitsWatcher(m)
}

// SetInstanceId sets the provider specific machine id for this machine.
func (m *Machine) SetInstanceId(id string) (err error) {
	defer errorContextf(&err, "can't set instance id of machine %s to %q", m, id)
	config, err := readConfigNode(m.st.zk, m.zkPath())
	if err != nil {
		return err
	}
	config.Set(providerMachineId, id)
	_, err = config.Write()
	return err
}

// String returns a unique description of this machine
func (m *Machine) String() string {
	return strconv.Itoa(m.Id())
}

// zkPath returns the ZooKeeper base path for the machine.
func (m *Machine) zkPath() string {
	return path.Join(zkMachinesPath, m.key)
}

// zkAgentPath returns the ZooKeeper path for the machine agent.
func (m *Machine) zkAgentPath() string {
	return path.Join(m.zkPath(), "agent")
}

// keySeq returns the sequence number part of
// the the given machine or unit key.
func keySeq(key string) (id int) {
	if key == "" {
		panic("keySeq: empty key")
	}
	i := strings.LastIndex(key, "-")
	var id64 int64
	var err error
	if i >= 0 {
		id64, err = strconv.ParseInt(key[i+1:], 10, 32)
	}
	if i < 0 || err != nil {
		panic("keySeq: invalid key: " + key)
	}
	return int(id64)
}

// machineKey returns the machine key corresponding to machineId.
func machineKey(machineId int) string {
	return fmt.Sprintf("machine-%010d", machineId)
}
<|endoftext|>


<commit_before>package database

import (
	"context"
	"fmt"

	"cloud.google.com/go/spanner"
	database "cloud.google.com/go/spanner/admin/database/apiv1"
	adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
	"google.golang.org/grpc/codes"
)

type Spanner struct {
	Database
	client *spanner.Client
	admin  *database.DatabaseAdminClient
}

// SetupSpanner initializes the spanner clients.
func NewSpanner(project, instance, db string) (*Spanner, error) {
	ctx := context.Background()
	sp := &Spanner{}

	adminClient, err := database.NewDatabaseAdminClient(ctx)
	if err != nil {
		return nil, err
	}

	sp.admin = adminClient

	// Create the databases if they don't exist.
	err = sp.createSpannerDatabase(ctx, project, instance, db)

	if err != nil {
		return nil, err
	}

	dbstr := fmt.Sprintf("projects/%s/instances/%s/databases/%s",
		project,
		instance,
		db)

	client, err := spanner.NewClient(context.Background(), dbstr)

	if err != nil {
		return nil, err
	}

	sp.client = client
	return sp, err
}

func (s *Spanner) Read() {
}

func (s *Spanner) Save(message *Message) error {
	ctx := context.WithValue(context.Background(), contextKey("message"), message)
	_, err := s.client.ReadWriteTransaction(ctx, s.doSave)
	return err
}

// Here's where the magic happens. Save out message!
func (s *Spanner) doSave(ctx context.Context, rw *spanner.ReadWriteTransaction) error {
	msg := ctx.Value(contextKey("message")).(*Message)

	// First, let's check and see if our message has been written.
	row, err := rw.ReadRow(context.Background(), "sheep_transaction", spanner.Key{msg.UUID}, []string{"applied"})
	if err != nil {
		if spanner.ErrCode(err) != codes.NotFound {
			return err
		}
	} else {
		var ap bool
		err = row.ColumnByName("Applied", &ap)
		if err != nil {
			return err
		}
		if ap {
			return nil
		}
	}

	// Let's get our current count
	row, err = rw.ReadRow(context.Background(), "sheep", spanner.Key{msg.Keyspace, msg.Key, msg.Name}, []string{"Count"})
	if err != nil {
		return err
	}
	var move int64
	row.ColumnByName("Count", &move)

	// Now we'll do our operation.
	switch msg.Operation {
	case "incr":
		move++
	case "decr":
		move--
	case "set":
		move = msg.Value
	default:
		return &spanner.Error{
			Desc: "Invalid operation sent from message, aborting transaction!",
		}
	}

	// Build our mutation...
	m := []*spanner.Mutation{
		spanner.InsertOrUpdate(
			"sheep_transaction",
			[]string{"UUID", "Applied"},
			[]interface{}{msg.UUID, true}),
		spanner.InsertOrUpdate(
			"sheep",
			[]string{"Keyspace", "Key", "Name", "Count"},
			[]interface{}{msg.Keyspace, msg.Key, msg.Name, move},
		),
	}

	// ...and write!
	return rw.BufferWrite(m)

}

func (s *Spanner) createSpannerDatabase(ctx context.Context, project, instance, db string) error {
	// Create our database if it doesn't exist.
	_, err := s.admin.GetDatabase(ctx, &adminpb.GetDatabaseRequest{
		Name: "projects/" + project + "/instances/" + instance + "/databases/" + db})
	if err != nil {
		// Database doesn't exist, or error.
		op, err := s.admin.CreateDatabase(ctx, &adminpb.CreateDatabaseRequest{
			Parent:          "projects/" + project + "/instances/" + instance,
			CreateStatement: "CREATE DATABASE `" + db + "`",
			ExtraStatements: []string{
				`CREATE TABLE sheep (
							Keyspace 	STRING(MAX) NOT NULL,
							Key 			STRING(MAX) NOT NULL,
							Name			STRING(MAX) NOT NULL,
							Count 		INT64
					) PRIMARY KEY (Keyspace, Key, Name)`,
				`CREATE TABLE sheep_transaction (
							UUID 			STRING(128) NOT NULL,
							Applied 	BOOL
					) PRIMARY KEY (UUID)`,
			},
		})

		if err != nil {
			return err
		}

		_, err = op.Wait(ctx)

		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>transaction table interleve<commit_after>package database

import (
	"context"
	"fmt"

	"cloud.google.com/go/spanner"
	database "cloud.google.com/go/spanner/admin/database/apiv1"
	adminpb "google.golang.org/genproto/googleapis/spanner/admin/database/v1"
	"google.golang.org/grpc/codes"
)

type Spanner struct {
	Database
	client *spanner.Client
	admin  *database.DatabaseAdminClient
}

// SetupSpanner initializes the spanner clients.
func NewSpanner(project, instance, db string) (*Spanner, error) {
	ctx := context.Background()
	sp := &Spanner{}

	adminClient, err := database.NewDatabaseAdminClient(ctx)
	if err != nil {
		return nil, err
	}

	sp.admin = adminClient

	// Create the databases if they don't exist.
	err = sp.createSpannerDatabase(ctx, project, instance, db)

	if err != nil {
		return nil, err
	}

	dbstr := fmt.Sprintf("projects/%s/instances/%s/databases/%s",
		project,
		instance,
		db)

	client, err := spanner.NewClient(context.Background(), dbstr)

	if err != nil {
		return nil, err
	}

	sp.client = client
	return sp, err
}

func (s *Spanner) Read() {
}

func (s *Spanner) Save(message *Message) error {
	ctx := context.WithValue(context.Background(), contextKey("message"), message)
	_, err := s.client.ReadWriteTransaction(ctx, s.doSave)
	return err
}

// Here's where the magic happens. Save out message!
func (s *Spanner) doSave(ctx context.Context, rw *spanner.ReadWriteTransaction) error {
	msg := ctx.Value(contextKey("message")).(*Message)

	// First, let's check and see if our message has been written.
	row, err := rw.ReadRow(context.Background(), "sheep_transaction", spanner.Key{msg.Keyspace, msg.Key, msg.Name, msg.UUID}, []string{"Applied"})
	if err != nil {
		if spanner.ErrCode(err) != codes.NotFound {
			return err
		}
	} else {
		var ap bool
		err = row.ColumnByName("Applied", &ap)
		if err != nil {
			return err
		}
		if ap {
			return nil
		}
	}

	// Let's get our current count
	var move int64
	row, err = rw.ReadRow(context.Background(), "sheep", spanner.Key{msg.Keyspace, msg.Key, msg.Name}, []string{"Count"})
	if err != nil {
		if spanner.ErrCode(err) != codes.NotFound {
			return err
		}
	} else {
		row.ColumnByName("Count", &move)
	}

	// Now we'll do our operation.
	switch msg.Operation {
	case "incr":
		move++
	case "decr":
		move--
	case "set":
		move = msg.Value
	default:
		return &spanner.Error{
			Desc: "Invalid operation sent from message, aborting transaction!",
		}
	}

	// Build our mutation...
	m := []*spanner.Mutation{
		spanner.InsertOrUpdate(
			"sheep",
			[]string{"Keyspace", "Key", "Name", "Count"},
			[]interface{}{msg.Keyspace, msg.Key, msg.Name, move},
		),
		spanner.InsertOrUpdate(
			"sheep_transaction",
			[]string{"Keyspace", "Key", "Name", "UUID", "Applied"},
			[]interface{}{msg.Keyspace, msg.Key, msg.Name, msg.UUID, true}),
	}

	// ...and write!
	return rw.BufferWrite(m)

}

func (s *Spanner) createSpannerDatabase(ctx context.Context, project, instance, db string) error {
	// Create our database if it doesn't exist.
	_, err := s.admin.GetDatabase(ctx, &adminpb.GetDatabaseRequest{
		Name: "projects/" + project + "/instances/" + instance + "/databases/" + db})
	if err != nil {
		// Database doesn't exist, or error.
		op, err := s.admin.CreateDatabase(ctx, &adminpb.CreateDatabaseRequest{
			Parent:          "projects/" + project + "/instances/" + instance,
			CreateStatement: "CREATE DATABASE `" + db + "`",
			ExtraStatements: []string{
				`CREATE TABLE sheep (
							Keyspace 	STRING(MAX) NOT NULL,
							Key 			STRING(MAX) NOT NULL,
							Name			STRING(MAX) NOT NULL,
							Count 		INT64
					) PRIMARY KEY (Keyspace, Key, Name)`,
				`CREATE TABLE sheep_transaction (
							Keyspace 	STRING(MAX) NOT NULL,
							Key 			STRING(MAX) NOT NULL,
							Name			STRING(MAX) NOT NULL,
							UUID 			STRING(128) NOT NULL,
							Applied 	BOOL
					) PRIMARY KEY (Keyspace, Key, Name, UUID),
						INTERLEAVE IN PARENT sheep ON DELETE CASCADE`,
			},
		})

		if err != nil {
			return err
		}

		_, err = op.Wait(ctx)

		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

// # TODO
// ------
// Using an ORM would have a huge advantage for the database fetcher as
// we will be able to use the same code over several DBMS.
// https://github.com/go-gorp/gorp
// gorp and other ORM are great but we need the ability to define our structure based
// on the configuration file.
// gorp support MySQL, PostgreSQL, sqlite3, Oracle & SQL Server

import (
	"bytes"
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/mattn/go-sqlite3"
	// "github.com/coopernurse/gorp"
	"errors"
	log "github.com/Sirupsen/logrus"
	"text/template"
)

// SQLFetcher is a database sql fetcher for CDRS, records will be retrieved
// from SQLFetcher and later pushed to the Pusher.
// SQLFetcher structure keeps tracks DB file, table, results and further data
// needed to fetch.
type SQLFetcher struct {
	db            *sql.DB
	DBFile        string
	DNS           string
	DBType        string
	DBTable       string
	DBFlagField   string
	MaxFetchBatch int
	numFetched    int
	cdrFields     []ParseFields
	results       map[int][]string
	sqlQuery      string
	listIDs       string
	IDField       string
}

// FetchSQL is used to build the SQL query to fetch on the Database source
type FetchSQL struct {
	ListFields string
	Table      string
	Limit      string
	Clause     string
	Order      string
}

// UpdateCDR is used to build the SQL query to update the Database source and
// track the records imported
type UpdateCDR struct {
	Table     string
	Fieldname string
	Status    int
	CDRids    string
	IDField   string
}

// Init is a constructor for SQLFetcher
// It will help setting DBFile, DBTable, MaxFetchBatch and cdrFields
func (f *SQLFetcher) Init(DBFile string, DBTable string, MaxFetchBatch int, cdrFields []ParseFields,
	DBFlagField string, DBType string, DNS string) {
	f.db = nil
	f.DBFile = DBFile
	f.DBTable = DBTable
	f.DBType = DBType
	f.DNS = DNS
	f.MaxFetchBatch = MaxFetchBatch
	f.numFetched = 0
	f.cdrFields = cdrFields
	f.results = nil
	f.sqlQuery = ""
	f.DBFlagField = DBFlagField
	f.IDField = "id"
}

// func NewSQLFetcher(DBFile string, DBTable string, MaxFetchBatch int, cdrFields []ParseFields) *SQLFetcher {
// 	db, _ := sql.Open("sqlite3", "./sqlitedb/cdr.db")
// 	return &SQLFetcher{db: db, DBFile: DBFile, DBTable: DBTable, sqlQuery: "", MaxFetchBatch, 0, cdrFields, nil}
// }

// Connect will help to connect to the DBMS, here we implemented the connection to SQLite
func (f *SQLFetcher) Connect() error {
	var err error
	if f.DBType == "sqlite3" {
		f.IDField = "rowid"
		f.db, err = sql.Open("sqlite3", f.DBFile)
		if err != nil {
			log.Error("Failed to connect", err)
			return err
		}
	} else if f.DBType == "mysql" {
		f.db, err = sql.Open("mysql", f.DNS)
		if err != nil {
			log.Error("Failed to connect", err)
			return err
		}
	} else {
		log.Error("DBType not supported!")
		return errors.New("DBType not supported!")
	}

	return nil
}

// PrepareQuery method will build the fetching SQL query
func (f *SQLFetcher) PrepareQuery() error {
	strFields := getFieldSelect(f.IDField, f.cdrFields)
	// parse the string cdrFields
	const tsql = "SELECT {{.ListFields}} FROM {{.Table}} {{.Clause}} {{.Order}} {{.Limit}}"
	var strSQL bytes.Buffer

	slimit := fmt.Sprintf("LIMIT %d", f.MaxFetchBatch)
	clause := "WHERE " + f.DBFlagField + "<>1"
	sqlb := FetchSQL{ListFields: strFields, Table: "cdr", Limit: slimit, Clause: clause}
	t := template.Must(template.New("sql").Parse(tsql))

	err := t.Execute(&strSQL, sqlb)
	if err != nil {
		panic(err)
	}
	f.sqlQuery = strSQL.String()
	log.Debug("SELECT_SQL: ", f.sqlQuery)
	return nil
}

// DBClose is helping defering the closing of the DB connector
func (f *SQLFetcher) DBClose() error {
	defer f.db.Close()
	return nil
}

// ScanResult method will scan the results and build the 2 propreties
// 'results' and 'listIDs'.
// - 'results' will held a map[int][]string that will contain all records
// - 'listIDs' will held a list of IDs from the results as a string
func (f *SQLFetcher) ScanResult() error {
	// Init numFetched to 0
	f.numFetched = 0
	rows, err := f.db.Query(f.sqlQuery)
	if err != nil {
		log.Error("Failed to run query:", err.Error())
		return err
	}
	defer rows.Close()
	cols, err := rows.Columns()
	if err != nil {
		log.Error("Failed to get columns:", err.Error())
		return err
	}
	// Result is your slice string.
	f.results = make(map[int][]string)
	listIDs := ""
	rawResult := make([][]byte, len(cols))
	result := make([]string, len(cols))

	dest := make([]interface{}, len(cols)) // A temporary interface{} slice
	for i := range rawResult {
		dest[i] = &rawResult[i] // Put pointers to each string in the interface slice
	}
	k := 0
	for rows.Next() {
		err = rows.Scan(dest...)
		if err != nil {
			log.Error("Failed to scan row", err)
			return err
		}
		for i, raw := range rawResult {
			if i == 0 {
				listIDs = listIDs + string(raw) + ", "
			}
			if raw == nil {
				result[i] = "\\N"
			} else {
				result[i] = string(raw)
			}
			f.results[k] = append(f.results[k], result[i])
		}
		k++
	}
	f.numFetched = k
	log.Info("Total fetched from Sqlite: ", f.numFetched)
	// Remove last ', ' from listIDs
	if listIDs != "" {
		f.listIDs = listIDs[0 : len(listIDs)-2]
	}
	return nil
}

// UpdateCdrTable method is used to mark the record that has been imported
func (f *SQLFetcher) UpdateCdrTable(status int) error {
	const tsql = "UPDATE {{.Table}} SET {{.Fieldname}}={{.Status}} WHERE {{.IDField}} IN ({{.CDRids}})"
	var strSQL bytes.Buffer

	sqlb := UpdateCDR{Table: f.DBTable, Fieldname: f.DBFlagField, Status: status, IDField: f.IDField, CDRids: f.listIDs}
	t := template.Must(template.New("sql").Parse(tsql))

	err := t.Execute(&strSQL, sqlb)
	log.Debug("UPDATE TABLE: ", &strSQL)
	if err != nil {
		return err
	}
	if _, err := f.db.Exec(strSQL.String()); err != nil {
		return err
	}
	return nil
}

// AddFieldTrackImport method will add a new field to your DB schema to track the import
func (f *SQLFetcher) AddFieldTrackImport() error {
	const tsql = "ALTER TABLE {{.Table}} ADD {{.Fieldname}} INTEGER DEFAULT 0"
	var strSQL bytes.Buffer

	sqlb := UpdateCDR{Table: f.DBTable, Fieldname: f.DBFlagField, Status: 0}
	t := template.Must(template.New("sql").Parse(tsql))

	err := t.Execute(&strSQL, sqlb)
	log.Debug("ALTER TABLE: ", &strSQL)
	if err != nil {
		return err
	}
	if _, err := f.db.Exec(strSQL.String()); err != nil {
		return err
	}
	return nil
}

// Fetch is the main method that will connect to the DB, add field for import tracking,
// prepare query and finally build the results
func (f *SQLFetcher) Fetch() error {
	// Connect to DB
	err := f.Connect()
	if err != nil {
		return err
	}
	defer f.db.Close()

	err = f.AddFieldTrackImport()
	if err != nil {
		log.Debug("Exec err (expected error if the field exist):", err.Error())
	}
	// Prepare SQL query
	err = f.PrepareQuery()
	if err != nil {
		return err
	}
	// Get Results
	err = f.ScanResult()
	if err != nil {
		return err
	}

	err = f.UpdateCdrTable(1)
	if err != nil {
		return err
	}
	log.Debug("RESULT:", f.results)
	return nil
}
<commit_msg>fix error to support custom table on import plus reset listIDs<commit_after>package main

// # TODO
// ------
// Using an ORM would have a huge advantage for the database fetcher as
// we will be able to use the same code over several DBMS.
// https://github.com/go-gorp/gorp
// gorp and other ORM are great but we need the ability to define our structure based
// on the configuration file.
// gorp support MySQL, PostgreSQL, sqlite3, Oracle & SQL Server

import (
	"bytes"
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/mattn/go-sqlite3"
	// "github.com/coopernurse/gorp"
	"errors"
	log "github.com/Sirupsen/logrus"
	"text/template"
)

// SQLFetcher is a database sql fetcher for CDRS, records will be retrieved
// from SQLFetcher and later pushed to the Pusher.
// SQLFetcher structure keeps tracks DB file, table, results and further data
// needed to fetch.
type SQLFetcher struct {
	db            *sql.DB
	DBFile        string
	DNS           string
	DBType        string
	DBTable       string
	DBFlagField   string
	MaxFetchBatch int
	numFetched    int
	cdrFields     []ParseFields
	results       map[int][]string
	sqlQuery      string
	listIDs       string
	IDField       string
}

// FetchSQL is used to build the SQL query to fetch on the Database source
type FetchSQL struct {
	ListFields string
	Table      string
	Limit      string
	Clause     string
	Order      string
}

// UpdateCDR is used to build the SQL query to update the Database source and
// track the records imported
type UpdateCDR struct {
	Table     string
	Fieldname string
	Status    int
	CDRids    string
	IDField   string
}

// Init is a constructor for SQLFetcher
// It will help setting DBFile, DBTable, MaxFetchBatch and cdrFields
func (f *SQLFetcher) Init(DBFile string, DBTable string, MaxFetchBatch int, cdrFields []ParseFields,
	DBFlagField string, DBType string, DNS string) {
	f.db = nil
	f.DBFile = DBFile
	f.DBTable = DBTable
	f.DBType = DBType
	f.DNS = DNS
	f.MaxFetchBatch = MaxFetchBatch
	f.numFetched = 0
	f.cdrFields = cdrFields
	f.results = nil
	f.sqlQuery = ""
	f.DBFlagField = DBFlagField
	f.IDField = "id"
}

// func NewSQLFetcher(DBFile string, DBTable string, MaxFetchBatch int, cdrFields []ParseFields) *SQLFetcher {
// 	db, _ := sql.Open("sqlite3", "./sqlitedb/cdr.db")
// 	return &SQLFetcher{db: db, DBFile: DBFile, DBTable: DBTable, sqlQuery: "", MaxFetchBatch, 0, cdrFields, nil}
// }

// Connect will help to connect to the DBMS, here we implemented the connection to SQLite
func (f *SQLFetcher) Connect() error {
	var err error
	if f.DBType == "sqlite3" {
		f.IDField = "rowid"
		f.db, err = sql.Open("sqlite3", f.DBFile)
		if err != nil {
			log.Error("Failed to connect", err)
			return err
		}
	} else if f.DBType == "mysql" {
		f.db, err = sql.Open("mysql", f.DNS)
		if err != nil {
			log.Error("Failed to connect", err)
			return err
		}
	} else {
		log.Error("DBType not supported!")
		return errors.New("DBType not supported!")
	}

	return nil
}

// PrepareQuery method will build the fetching SQL query
func (f *SQLFetcher) PrepareQuery() error {
	strFields := getFieldSelect(f.IDField, f.cdrFields)
	// parse the string cdrFields
	const tsql = "SELECT {{.ListFields}} FROM {{.Table}} {{.Clause}} {{.Order}} {{.Limit}}"
	var strSQL bytes.Buffer

	slimit := fmt.Sprintf("LIMIT %d", f.MaxFetchBatch)
	clause := "WHERE " + f.DBFlagField + "<>1"
	sqlb := FetchSQL{ListFields: strFields, Table: f.DBTable, Limit: slimit, Clause: clause}
	t := template.Must(template.New("sql").Parse(tsql))

	err := t.Execute(&strSQL, sqlb)
	if err != nil {
		panic(err)
	}
	f.sqlQuery = strSQL.String()
	log.Debug("SELECT_SQL: ", f.sqlQuery)
	return nil
}

// DBClose is helping defering the closing of the DB connector
func (f *SQLFetcher) DBClose() error {
	defer f.db.Close()
	return nil
}

// ScanResult method will scan the results and build the 2 propreties
// 'results' and 'listIDs'.
// - 'results' will held a map[int][]string that will contain all records
// - 'listIDs' will held a list of IDs from the results as a string
func (f *SQLFetcher) ScanResult() error {
	// Init numFetched to 0
	f.numFetched = 0
	rows, err := f.db.Query(f.sqlQuery)
	if err != nil {
		log.Error("Failed to run query:", err.Error())
		return err
	}
	defer rows.Close()
	cols, err := rows.Columns()
	if err != nil {
		log.Error("Failed to get columns:", err.Error())
		return err
	}
	// Result is your slice string.
	f.results = make(map[int][]string)
	f.listIDs = ""
	rawResult := make([][]byte, len(cols))
	result := make([]string, len(cols))

	dest := make([]interface{}, len(cols)) // A temporary interface{} slice
	for i := range rawResult {
		dest[i] = &rawResult[i] // Put pointers to each string in the interface slice
	}
	k := 0
	for rows.Next() {
		err = rows.Scan(dest...)
		if err != nil {
			log.Error("Failed to scan row", err)
			return err
		}
		for i, raw := range rawResult {
			if i == 0 {
				f.listIDs = f.listIDs + string(raw) + ", "
			}
			if raw == nil {
				result[i] = "\\N"
			} else {
				result[i] = string(raw)
			}
			f.results[k] = append(f.results[k], result[i])
		}
		k++
	}
	f.numFetched = k
	log.Info("Total fetched from database: ", f.numFetched)
	// Remove last ', ' from f.listIDs
	if f.listIDs != "" {
		f.listIDs = f.listIDs[0 : len(f.listIDs)-2]
	}
	return nil
}

// UpdateCdrTable method is used to mark the record that has been imported
func (f *SQLFetcher) UpdateCdrTable(status int) error {
	const tsql = "UPDATE {{.Table}} SET {{.Fieldname}}={{.Status}} WHERE {{.IDField}} IN ({{.CDRids}})"
	var strSQL bytes.Buffer

	if len(f.listIDs) > 0 {
		sqlb := UpdateCDR{Table: f.DBTable, Fieldname: f.DBFlagField, Status: status, IDField: f.IDField, CDRids: f.listIDs}
		t := template.Must(template.New("sql").Parse(tsql))

		err := t.Execute(&strSQL, sqlb)
		log.Debug("UPDATE TABLE: ", &strSQL)
		if err != nil {
			return err
		}
		if _, err := f.db.Exec(strSQL.String()); err != nil {
			return err
		}
	} else {
		log.Debug("No IDs to update...")
	}
	return nil
}

// AddFieldTrackImport method will add a new field to your DB schema to track the import
func (f *SQLFetcher) AddFieldTrackImport() error {
	const tsql = "ALTER TABLE {{.Table}} ADD {{.Fieldname}} INTEGER DEFAULT 0"
	var strSQL bytes.Buffer

	sqlb := UpdateCDR{Table: f.DBTable, Fieldname: f.DBFlagField, Status: 0}
	t := template.Must(template.New("sql").Parse(tsql))

	err := t.Execute(&strSQL, sqlb)
	log.Debug("ALTER TABLE: ", &strSQL)
	if err != nil {
		return err
	}
	if _, err := f.db.Exec(strSQL.String()); err != nil {
		return err
	}
	return nil
}

// Fetch is the main method that will connect to the DB, add field for import tracking,
// prepare query and finally build the results
func (f *SQLFetcher) Fetch() error {
	// Connect to DB
	err := f.Connect()
	if err != nil {
		return err
	}
	defer f.db.Close()

	err = f.AddFieldTrackImport()
	if err != nil {
		log.Debug("Exec err (expected error if the field exist):", err.Error())
	}
	// Prepare SQL query
	err = f.PrepareQuery()
	if err != nil {
		return err
	}
	// Get Results
	err = f.ScanResult()
	if err != nil {
		return err
	}

	err = f.UpdateCdrTable(1)
	if err != nil {
		return err
	}
	log.Debug("RESULT:", f.results)
	return nil
}
<|endoftext|>


<commit_before>package brightbox

import (
	"time"
)

// DatabaseServer represents a database server.
// https://api.gb1.brightbox.com/1.0/#database_server
type DatabaseServer struct {
	Id                 string
	Name               string
	Description        string
	Status             string
	Account            Account
	DatabaseEngine     string     `json:"database_engine"`
	DatabaseVersion    string     `json:"database_version"`
	AdminUsername      string     `json:"admin_username"`
	AdminPassword      string     `json:"admin_password"`
	CreatedAt          *time.Time `json:"created_at"`
	UpdatedAt          *time.Time `json:"updated_at"`
	DeletedAt          *time.Time `json:"deleted_at"`
	AllowAccess        []string   `json:"allow_access"`
	MaintenanceWeekday int        `json:"maintenance_weekday"`
	MaintenanceHour    int        `json:"maintenance_hour"`
	Locked             bool
	CloudIPs           []CloudIP `json:"cloud_ips"`
	Zone               Zone
	DatabaseServerType DatabaseServerType `json:"database_server_type"`
}

// DatabaseServerOptions is used in conjunction with CreateDatabaseServer and
// UpdateDatabaseServer to create and update database servers.
type DatabaseServerOptions struct {
	Id                 string   `json:"-"`
	Name               *string  `json:"name,omitempty"`
	Description        *string  `json:"description,omitempty"`
	Engine             string   `json:"engine,omitempty"`
	Version            string   `json:"version,omitempty"`
	AllowAccess        []string `json:"allow_access,omitempty"`
	Snapshot           string   `json:"snapshot,omitempty"`
	Zone               string   `json:"zone,omitempty"`
	DatabaseType       string   `json:"database_type,omitempty"`
	MaintenanceWeekday *int     `json:"maintenance_weekday,omitempty"`
	MaintenanceHour    *int     `json:"maintenance_hour,omitempty"`
}

// DatabaseServers retrieves a list of all database servers
func (c *Client) DatabaseServers() ([]DatabaseServer, error) {
	var dbs []DatabaseServer
	_, err := c.MakeApiRequest("GET", "/1.0/database_servers", nil, &dbs)
	if err != nil {
		return nil, err
	}
	return dbs, err
}

// DatabaseServer retrieves a detailed view of one database server
func (c *Client) DatabaseServer(identifier string) (*DatabaseServer, error) {
	dbs := new(DatabaseServer)
	_, err := c.MakeApiRequest("GET", "/1.0/database_servers/"+identifier, nil, dbs)
	if err != nil {
		return nil, err
	}
	return dbs, err
}

// CreateDatabaseServer creates a new database server.
//
// It takes a DatabaseServerOptions struct for specifying name and other
// attributes. Not all attributes can be specified at create time
// (such as Id, which is allocated for you)
func (c *Client) CreateDatabaseServer(options *DatabaseServerOptions) (*DatabaseServer, error) {
	dbs := new(DatabaseServer)
	_, err := c.MakeApiRequest("POST", "/1.0/database_servers", options, &dbs)
	if err != nil {
		return nil, err
	}
	return dbs, nil
}

// UpdateDatabaseServer updates an existing database server.
//
// It takes a DatabaseServerOptions struct for specifying Id, name and other
// attributes. Not all attributes can be specified at update time.
func (c *Client) UpdateDatabaseServer(options *DatabaseServerOptions) (*DatabaseServer, error) {
	dbs := new(DatabaseServer)
	_, err := c.MakeApiRequest("PUT", "/1.0/database_servers/"+options.Id, options, &dbs)
	if err != nil {
		return nil, err
	}
	return dbs, nil
}

// DestroyDatabaseServer issues a request to deletes an existing database server
func (c *Client) DestroyDatabaseServer(identifier string) error {
	_, err := c.MakeApiRequest("DELETE", "/1.0/database_servers/"+identifier, nil, nil)
	return err
}

// SnapshotDatabaseServer requests a snapshot of an existing database server.
func (c *Client) SnapshotDatabaseServer(identifier string) (*DatabaseSnapshot, error) {
	dbs := new(DatabaseServer)
	res, err := c.MakeApiRequest("POST", "/1.0/database_servers/"+identifier+"/snapshot", nil, &dbs)
	if err != nil {
		return nil, err
	}
	snapID := getLinkRel(res.Header.Get("Link"), "dbi", "snapshot")
	if snapID != nil {
		snap := new(DatabaseSnapshot)
		snap.Id = *snapID
		return snap, nil
	}
	return nil, nil
}

// ResetPasswordForDatabaseServer requests a snapshot of an existing database server.
func (c *Client) ResetPasswordForDatabaseServer(identifier string) (*DatabaseServer, error) {
	dbs := new(DatabaseServer)
	_, err := c.MakeApiRequest("POST", "/1.0/database_servers/"+identifier+"/reset_password", nil, &dbs)
	if err != nil {
		return nil, err
	}
	return dbs, nil
}
<commit_msg>Add snapshot schedule to database server structures<commit_after>package brightbox

import (
	"time"
)

// DatabaseServer represents a database server.
// https://api.gb1.brightbox.com/1.0/#database_server
type DatabaseServer struct {
	Id                 string
	Name               string
	Description        string
	Status             string
	Account            Account
	DatabaseEngine     string             `json:"database_engine"`
	DatabaseVersion    string             `json:"database_version"`
	AdminUsername      string             `json:"admin_username"`
	AdminPassword      string             `json:"admin_password"`
	CreatedAt          *time.Time         `json:"created_at"`
	UpdatedAt          *time.Time         `json:"updated_at"`
	DeletedAt          *time.Time         `json:"deleted_at"`
	AllowAccess        []string           `json:"allow_access"`
	MaintenanceWeekday int                `json:"maintenance_weekday"`
	MaintenanceHour    int                `json:"maintenance_hour"`
	SnapshotsSchedule  string             `json:"snapshots_schedule"`
	CloudIPs           []CloudIP          `json:"cloud_ips"`
	DatabaseServerType DatabaseServerType `json:"database_server_type"`
	Locked             bool
	Zone               Zone
}

// DatabaseServerOptions is used in conjunction with CreateDatabaseServer and
// UpdateDatabaseServer to create and update database servers.
type DatabaseServerOptions struct {
	Id                 string   `json:"-"`
	Name               *string  `json:"name,omitempty"`
	Description        *string  `json:"description,omitempty"`
	Engine             string   `json:"engine,omitempty"`
	Version            string   `json:"version,omitempty"`
	AllowAccess        []string `json:"allow_access,omitempty"`
	Snapshot           string   `json:"snapshot,omitempty"`
	Zone               string   `json:"zone,omitempty"`
	DatabaseType       string   `json:"database_type,omitempty"`
	MaintenanceWeekday *int     `json:"maintenance_weekday,omitempty"`
	MaintenanceHour    *int     `json:"maintenance_hour,omitempty"`
	SnapshotsSchedule  *string  `json:"snapshots_schedule,omitempty"`
}

// DatabaseServers retrieves a list of all database servers
func (c *Client) DatabaseServers() ([]DatabaseServer, error) {
	var dbs []DatabaseServer
	_, err := c.MakeApiRequest("GET", "/1.0/database_servers", nil, &dbs)
	if err != nil {
		return nil, err
	}
	return dbs, err
}

// DatabaseServer retrieves a detailed view of one database server
func (c *Client) DatabaseServer(identifier string) (*DatabaseServer, error) {
	dbs := new(DatabaseServer)
	_, err := c.MakeApiRequest("GET", "/1.0/database_servers/"+identifier, nil, dbs)
	if err != nil {
		return nil, err
	}
	return dbs, err
}

// CreateDatabaseServer creates a new database server.
//
// It takes a DatabaseServerOptions struct for specifying name and other
// attributes. Not all attributes can be specified at create time
// (such as Id, which is allocated for you)
func (c *Client) CreateDatabaseServer(options *DatabaseServerOptions) (*DatabaseServer, error) {
	dbs := new(DatabaseServer)
	_, err := c.MakeApiRequest("POST", "/1.0/database_servers", options, &dbs)
	if err != nil {
		return nil, err
	}
	return dbs, nil
}

// UpdateDatabaseServer updates an existing database server.
//
// It takes a DatabaseServerOptions struct for specifying Id, name and other
// attributes. Not all attributes can be specified at update time.
func (c *Client) UpdateDatabaseServer(options *DatabaseServerOptions) (*DatabaseServer, error) {
	dbs := new(DatabaseServer)
	_, err := c.MakeApiRequest("PUT", "/1.0/database_servers/"+options.Id, options, &dbs)
	if err != nil {
		return nil, err
	}
	return dbs, nil
}

// DestroyDatabaseServer issues a request to deletes an existing database server
func (c *Client) DestroyDatabaseServer(identifier string) error {
	_, err := c.MakeApiRequest("DELETE", "/1.0/database_servers/"+identifier, nil, nil)
	return err
}

// SnapshotDatabaseServer requests a snapshot of an existing database server.
func (c *Client) SnapshotDatabaseServer(identifier string) (*DatabaseSnapshot, error) {
	dbs := new(DatabaseServer)
	res, err := c.MakeApiRequest("POST", "/1.0/database_servers/"+identifier+"/snapshot", nil, &dbs)
	if err != nil {
		return nil, err
	}
	snapID := getLinkRel(res.Header.Get("Link"), "dbi", "snapshot")
	if snapID != nil {
		snap := new(DatabaseSnapshot)
		snap.Id = *snapID
		return snap, nil
	}
	return nil, nil
}

// ResetPasswordForDatabaseServer requests a snapshot of an existing database server.
func (c *Client) ResetPasswordForDatabaseServer(identifier string) (*DatabaseServer, error) {
	dbs := new(DatabaseServer)
	_, err := c.MakeApiRequest("POST", "/1.0/database_servers/"+identifier+"/reset_password", nil, &dbs)
	if err != nil {
		return nil, err
	}
	return dbs, nil
}
<|endoftext|>


<commit_before>/*
	Go Language Raspberry Pi Interface
	(c) Copyright David Thorpe 2016-2018
	All Rights Reserved
	Documentation http://djthorpe.github.io/gopi/
	For Licensing and Usage information, please see LICENSE.md
*/

package gopi

import "strings"

////////////////////////////////////////////////////////////////////////////////
// TYPES

// SurfaceType of surface (which API it's bound to)
type SurfaceType uint

// SurfaceFlags are flags associated with surface
// usually during operations
type SurfaceFlags uint32

////////////////////////////////////////////////////////////////////////////////
// INTERFACES

// SurfaceManager allows you to open, close and move
// surfaces around an open display
type SurfaceManager interface {
	Driver

	// Return the display associated with the surface manager
	Display() Display

	// Return the name of the surface manager. It's basically the
	// GPU driver
	Name() string

	// Return capabilities for the GPU
	Types() []SurfaceType

	// Create & destroy surfaces
	CreateSurface(api SurfaceType, flags SurfaceFlags, opacity float32, layer uint, origin Point, size Size) (Surface, error)
	DestroySurface(Surface) error

	/*
		// Create background, surface and cursors
		CreateBackground(api SurfaceType, flags SurfaceFlags, opacity float32) (Surface, error)
		CreateCursor(api SurfaceType, flags SurfaceFlags, opacity float32, origin Point, cursor SurfaceCursor) (Surface, error)

		// Change surface properties (size, position, etc)
		MoveOriginBy(Surface, SurfaceFlags, Point)
		SetOrigin(Surface, SurfaceFlags, Point)
		SetSize(Surface, SurfaceFlags, Size)
		SetOpacity(Surface, SurfaceFlags, float32)
		SetLayer(Surface, uint)

		// Surface operations to start and end drawing or other
		// surface operations
		SetCurrentContext(Surface)
		FlushSurface(Surface)
	*/
}

// Surface is manipulated by surface manager, and used by
// a GPU API (bitmap or vector drawing mostly)
type Surface interface {
	Type() SurfaceType
	Opacity() float32
	Layer() uint
	Origin() Point
	Size() Size
}

/*
type SurfaceCursor interface {
	API()
	Hotspot()
	Size()
}
*/

////////////////////////////////////////////////////////////////////////////////
// CONSTANTS

const (
	// SurfaceType
	SURFACE_TYPE_NONE SurfaceType = iota
	SURFACE_TYPE_OPENGL
	SURFACE_TYPE_OPENGL_ES
	SURFACE_TYPE_OPENGL_ES2
	SURFACE_TYPE_OPENVG
	SURFACE_TYPE_RGBA32
)

const (
	// SurfaceType
	SURFACE_FLAG_NONE SurfaceFlags = 0
)

////////////////////////////////////////////////////////////////////////////////
// STRINGIFY

func (t SurfaceType) String() string {
	switch t {
	case SURFACE_TYPE_OPENGL:
		return "SURFACE_TYPE_OPENGL"
	case SURFACE_TYPE_OPENGL_ES:
		return "SURFACE_TYPE_OPENGL_ES"
	case SURFACE_TYPE_OPENGL_ES2:
		return "SURFACE_TYPE_OPENGL_ES2"
	case SURFACE_TYPE_OPENVG:
		return "SURFACE_TYPE_OPENVG"
	case SURFACE_TYPE_RGBA32:
		return "SURFACE_TYPE_RGBA32"
	default:
		return "[Invalid SurfaceType value]"
	}
}

func (f SurfaceFlags) String() string {
	if f == SURFACE_FLAG_NONE {
		return "SURFACE_FLAG_NONE"
	}
	flags := ""
	// Add flags here
	return strings.Trim(flags, "|")
}
<commit_msg>Updated graphics and made a few name changes<commit_after>/*
	Go Language Raspberry Pi Interface
	(c) Copyright David Thorpe 2016-2018
	All Rights Reserved
	Documentation http://djthorpe.github.io/gopi/
	For Licensing and Usage information, please see LICENSE.md
*/

package gopi

import "strings"

////////////////////////////////////////////////////////////////////////////////
// TYPES

// SurfaceType of surface (which API it's bound to)
type SurfaceType uint

// SurfaceFlags are flags associated with surface
// usually during operations
type SurfaceFlags uint32

////////////////////////////////////////////////////////////////////////////////
// INTERFACES

// SurfaceManager allows you to open, close and move
// surfaces around an open display
type SurfaceManager interface {
	Driver

	// Return the display associated with the surface manager
	Display() Display

	// Return the name of the surface manager. It's basically the
	// GPU driver
	Name() string

	// Return capabilities for the GPU
	Types() []SurfaceType

	// Create & destroy surfaces
	CreateSurface(api SurfaceType, flags SurfaceFlags, opacity float32, layer uint, origin Point, size Size) (Surface, error)
	DestroySurface(Surface) error

	// Create and destroy bitmaps
	CreateBitmap(Size) (Bitmap, error)
	DestroyBitmap(Bitmap) error

	/*
		// Create background, surface and cursors
		CreateBackground(api SurfaceType, flags SurfaceFlags, opacity float32) (Surface, error)
		CreateCursor(api SurfaceType, flags SurfaceFlags, opacity float32, origin Point, cursor SurfaceCursor) (Surface, error)

		// Change surface properties (size, position, etc)
		MoveOriginBy(Surface, SurfaceFlags, Point)
		SetOrigin(Surface, SurfaceFlags, Point)
		SetSize(Surface, SurfaceFlags, Size)
		SetOpacity(Surface, SurfaceFlags, float32)
		SetLayer(Surface, uint)

		// Surface operations to start and end drawing or other
		// surface operations
		SetCurrentContext(Surface)
		FlushSurface(Surface)
	*/
}

// Surface is manipulated by surface manager, and used by
// a GPU API (bitmap or vector drawing mostly)
type Surface interface {
	Type() SurfaceType
	Opacity() float32
	Layer() uint
	Origin() Point
	Size() Size
}

// Bitmap defines a rectangular bitmap which can be used
// by the GPU
type Bitmap interface {
	Size() Size
}

/*
type SurfaceCursor interface {
	API()
	Hotspot()
	Size()
}
*/

////////////////////////////////////////////////////////////////////////////////
// CONSTANTS

const (
	// SurfaceType
	SURFACE_TYPE_NONE SurfaceType = iota
	SURFACE_TYPE_OPENGL
	SURFACE_TYPE_OPENGL_ES
	SURFACE_TYPE_OPENGL_ES2
	SURFACE_TYPE_OPENVG
	SURFACE_TYPE_RGBA32
)

const (
	// SurfaceType
	SURFACE_FLAG_NONE SurfaceFlags = 0
)

////////////////////////////////////////////////////////////////////////////////
// STRINGIFY

func (t SurfaceType) String() string {
	switch t {
	case SURFACE_TYPE_OPENGL:
		return "SURFACE_TYPE_OPENGL"
	case SURFACE_TYPE_OPENGL_ES:
		return "SURFACE_TYPE_OPENGL_ES"
	case SURFACE_TYPE_OPENGL_ES2:
		return "SURFACE_TYPE_OPENGL_ES2"
	case SURFACE_TYPE_OPENVG:
		return "SURFACE_TYPE_OPENVG"
	case SURFACE_TYPE_RGBA32:
		return "SURFACE_TYPE_RGBA32"
	default:
		return "[Invalid SurfaceType value]"
	}
}

func (f SurfaceFlags) String() string {
	if f == SURFACE_FLAG_NONE {
		return "SURFACE_FLAG_NONE"
	}
	flags := ""
	// Add flags here
	return strings.Trim(flags, "|")
}
<|endoftext|>


<commit_before>package svn

import (
	"encoding/xml"

	"github.com/pkg/errors"
)

type Log struct {
	XMLName xml.Name   `xml:"log"`
	Entries []LogEntry `xml:"logentry"`
}

type LogEntry struct {
	*Commit
	XMLName xml.Name `xml:"logentry"`
	Paths   Paths    `xml:"paths"`
	Message string   `xml:"msg"`
}

type Paths struct {
	XMLName xml.Name `xml:"paths"`
	Paths   []Path   `xml:"path"`
}

type Path struct {
	XMLName               xml.Name `xml:"path"`
	TextModifications     bool     `xml:"text-mods,attr"`
	Kind                  string   `xml:"kind,attr"`
	CopyFromPath          *string  `xml:"copyfrom-path,attr,omitempty"`
	CopyFromRevision      *int     `xml:"copyfrom-rev,attr,omitempty"`
	Action                string   `xml:"action,attr"`
	PropertyModifications bool     `xml:"prop-mods,attr"`
}

func GetLog(address string) (*Log, error) {
	log := Log{}

	if err := Execute(&log, "log", "--xml", "--verbose", address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s", address)
	}

	return &log, nil
}
<commit_msg>Remove intermediate Paths type<commit_after>package svn

import (
	"encoding/xml"

	"github.com/pkg/errors"
)

type Log struct {
	XMLName xml.Name   `xml:"log"`
	Entries []LogEntry `xml:"logentry"`
}

type LogEntry struct {
	*Commit
	XMLName xml.Name `xml:"logentry"`
	Paths   []Path   `xml:"paths>path"`
	Message string   `xml:"msg"`
}

type Path struct {
	XMLName               xml.Name `xml:"path"`
	TextModifications     bool     `xml:"text-mods,attr"`
	Kind                  string   `xml:"kind,attr"`
	CopyFromPath          *string  `xml:"copyfrom-path,attr,omitempty"`
	CopyFromRevision      *int     `xml:"copyfrom-rev,attr,omitempty"`
	Action                string   `xml:"action,attr"`
	PropertyModifications bool     `xml:"prop-mods,attr"`
}

func GetLog(address string) (*Log, error) {
	log := Log{}

	if err := Execute(&log, "log", "--xml", "--verbose", address); err != nil {
		return nil, errors.Wrapf(err, "failed to get log for %s", address)
	}

	return &log, nil
}
<|endoftext|>


<commit_before>package storage

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"net/url"
)

type MySQLStorage struct {
	db     *sql.DB
	events chan *ChangeSet
}

func init() {
	register["mysql"] = &MySQLStorage{}
}

func (s *MySQLStorage) Init(u *url.URL, c chan *ChangeSet) error {
	s.events = c

	// Build MySQL connection
	var err error
	// Strip "mysql://" from DSN
	s.db, err = sql.Open(u.Scheme, u.String()[8:])
	if err != nil {
		return err
	}

	// Open doesn't open a connection. Validate DSN data
	err = s.db.Ping()
	if err != nil {
		return err
	}

	return nil
}

func (s *MySQLStorage) Listen() {
	for c := range s.events {
		log.Println("MySQLStorage: New Event", c)
	}
}

func (s *MySQLStorage) Close() error {
	s.db.Close()
	return nil
}
<commit_msg>Added TODO to consume MySQL events<commit_after>package storage

import (
	"database/sql"
	_ "github.com/go-sql-driver/mysql"
	"log"
	"net/url"
)

type MySQLStorage struct {
	db     *sql.DB
	events chan *ChangeSet
}

func init() {
	register["mysql"] = &MySQLStorage{}
}

func (s *MySQLStorage) Init(u *url.URL, c chan *ChangeSet) error {
	s.events = c

	// Build MySQL connection
	var err error
	// Strip "mysql://" from DSN
	s.db, err = sql.Open(u.Scheme, u.String()[8:])
	if err != nil {
		return err
	}

	// Open doesn't open a connection. Validate DSN data
	err = s.db.Ping()
	if err != nil {
		return err
	}

	return nil
}

func (s *MySQLStorage) Listen() {
	for c := range s.events {
		// TODO Consume events and store them
		log.Println("MySQLStorage: New Event", c)
	}
}

func (s *MySQLStorage) Close() error {
	s.db.Close()
	return nil
}
<|endoftext|>