<commit_before>package monitor

import (
	"encoding/json"
	"github.com/shirou/gopsutil/v3/cpu"
	"github.com/shirou/gopsutil/v3/disk"
	"github.com/shirou/gopsutil/v3/host"
	"github.com/shirou/gopsutil/v3/load"
	"github.com/shirou/gopsutil/v3/mem"
	"github.com/shirou/gopsutil/v3/net"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_auth"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn"
	mo_path2 "github.com/watermint/toolbox/domain/dropbox/model/mo_path"
	"github.com/watermint/toolbox/domain/dropbox/model/mo_time"
	"github.com/watermint/toolbox/domain/dropbox/service/sv_file_content"
	"github.com/watermint/toolbox/essentials/file/es_filepath"
	"github.com/watermint/toolbox/essentials/file/es_gzip"
	"github.com/watermint/toolbox/essentials/log/esl"
	"github.com/watermint/toolbox/essentials/model/mo_int"
	"github.com/watermint/toolbox/essentials/model/mo_path"
	"github.com/watermint/toolbox/infra/app"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/quality/infra/qt_file"
	"github.com/watermint/toolbox/quality/recipe/qtr_endtoend"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

const (
	monitorFilePrefix = "tbx-monitor-"
)

type Client struct {
	DataPath        mo_path.FileSystemPath
	SyncPath        mo_path2.DropboxPath
	Peer            dbx_conn.ConnScopedIndividual
	Name            string
	MonitorInterval mo_int.RangeInt
	SyncInterval    mo_int.RangeInt
	MonitorEnd      mo_time.TimeOptional
	Display         bool

	sentErrors       map[string]bool
	currentJournal   *os.File
	currentPath      string
	currentStart     time.Time
	currentDeadline  time.Time
	rotateInProgress bool
}

func (z *Client) Preset() {
	z.MonitorInterval.SetRange(1, 86400, 10)
	z.SyncInterval.SetRange(10, 86400, 3600)
	z.sentErrors = make(map[string]bool)
	z.Peer.SetScopes(
		dbx_auth.ScopeFilesContentWrite,
	)
}
func (z *Client) sendError(c app_control.Control, eventType string, err error) {
	if _, ok := z.sentErrors[eventType]; ok {
		return
	}
	l := c.Log()
	l.Warn("Unable to retrieve event data", esl.String("type", eventType), esl.Error(err))
	z.sentErrors[eventType] = true
}

func (z *Client) openJournal(c app_control.Control) error {
	l := c.Log()
	if z.currentJournal != nil {
		return nil
	}
	name := monitorFilePrefix + es_filepath.Escape(z.Name) + "-" + strconv.FormatInt(time.Now().Unix(), 16) + ".log"
	path := filepath.Join(z.DataPath.Path(), name)
	l = l.With(esl.String("path", path))
	f, err := os.Create(path)
	if err != nil {
		l.Debug("Unable to create the log file", esl.Error(err))
		return err
	}
	z.currentJournal = f
	z.currentPath = path
	z.currentStart = time.Now()
	z.currentDeadline = z.currentStart.Add(time.Duration(z.SyncInterval.Value()) * time.Second)
	l.Debug("Journal created", esl.Time("deadline", z.currentDeadline))
	return nil
}

func (z *Client) syncJournal(c app_control.Control) error {
	l := c.Log()
	if err := z.currentJournal.Close(); err != nil {
		l.Debug("Unable to close", esl.Error(err))
	}
	if _, err := es_gzip.Compress(z.currentPath); err != nil {
		l.Debug("Unable to compress", esl.Error(err))
		return err
	}
	z.currentJournal = nil
	z.currentPath = ""

	files, err := os.ReadDir(z.DataPath.Path())
	if err != nil {
		l.Debug("Unable to read directory entry")
		return err
	}

	sv := sv_file_content.NewUpload(z.Peer.Client())
	basePath := z.SyncPath.ChildPath(es_filepath.Escape(z.Name), z.currentStart.Format("2006-01"), z.currentStart.Format("2006-01-02"))
	for _, f := range files {
		if !strings.HasPrefix(f.Name(), monitorFilePrefix) {
			continue
		}
		fp := filepath.Join(z.DataPath.Path(), f.Name())
		l.Info("Syncing journal file", esl.String("name", f.Name()))
		entry, err := sv.Add(basePath, fp)
		if err != nil {
			l.Debug("Unable to upload", esl.Error(err))
			return err
		}
		l.Debug("Upload completed", esl.Any("entry", entry))
		_ = os.Remove(fp)
	}
	return nil
}

func (z *Client) sendEvent(c app_control.Control, eventType string, data interface{}) {
	ev := Event{
		Time: time.Now().Format(time.RFC3339),
		Type: eventType,
		Data: data,
	}
	evs, err := json.Marshal(&ev)
	if err != nil {
		z.sendError(c, eventType, err)
		return
	}
	if z.Display {
		c.Log().Info("event", esl.Any("data", ev))
	}

	if z.currentJournal != nil {
		_, err0 := z.currentJournal.Write(evs)
		_, err1 := z.currentJournal.Write([]byte("\n"))

		if z.rotateInProgress {
			return
		}

		if err0 == nil && err1 == nil && z.currentDeadline.Before(time.Now()) {
			z.rotateInProgress = true
			if err := z.syncJournal(c); err != nil {
				z.sendError(c, eventType, err)
			}
			if err := z.openJournal(c); err != nil {
				z.sendError(c, eventType, err)
			}
			z.headEvents(c)
			z.rotateInProgress = false
		}
	}
}

func (z *Client) headEventCpuInfo(c app_control.Control) {
	cpuInfo, err := cpu.Info()
	if err != nil {
		z.sendError(c, EventCpuInfo, err)
		return
	}
	z.sendEvent(c, EventCpuInfo, cpuInfo)
}

func (z *Client) eventCpuTime(c app_control.Control) {
	stat, err := cpu.Times(true)
	if err != nil {
		z.sendError(c, EventCpuTime, err)
		return
	}
	z.sendEvent(c, EventCpuTime, stat)
}

func (z *Client) eventCpuPercent(c app_control.Control) {
	stat, err := cpu.Percent(0, true)
	if err != nil {
		z.sendError(c, EventCpuPercent, err)
		return
	}
	z.sendEvent(c, EventCpuPercent, stat)
}

func (z *Client) headEventHostInfo(c app_control.Control) {
	hostInfo, err := host.Info()
	if err != nil {
		z.sendError(c, EventHostInfo, err)
		return
	}
	z.sendEvent(c, EventHostInfo, hostInfo)
}

func (z *Client) headEventDiskPartition(c app_control.Control) {
	info, err := disk.Partitions(true)
	if err != nil {
		z.sendError(c, EventDiskPartition, err)
		return
	}
	z.sendEvent(c, EventDiskPartition, info)
}

func (z *Client) eventDiskUsage(c app_control.Control) {
	partitions, err := disk.Partitions(true)
	if err != nil {
		z.sendError(c, EventDiskUsage, err)
		return
	}
	for _, p := range partitions {
		usage, err := disk.Usage(p.Mountpoint)
		if err != nil {
			z.sendError(c, EventDiskUsage, err)
			continue
		}
		z.sendEvent(c, EventDiskUsage, usage)
	}
}

func (z *Client) eventLoadAverage(c app_control.Control) {
	la, err := load.Avg()
	if err != nil {
		z.sendError(c, EventLoadAverage, err)
		return
	}
	z.sendEvent(c, EventLoadAverage, la)
}

func (z *Client) eventMemoryStat(c app_control.Control) {
	vm, err := mem.VirtualMemory()
	if err != nil {
		z.sendError(c, EventMemoryStat, err)
		return
	}
	z.sendEvent(c, EventMemoryStat, vm)
}

func (z *Client) eventNetIO(c app_control.Control) {
	stats, err := net.IOCounters(true)
	if err != nil {
		z.sendError(c, EventNetIO, err)
		return
	}
	z.sendEvent(c, EventNetIO, stats)
}

func (z *Client) eventNetProtocol(c app_control.Control) {
	stats, err := net.ProtoCounters([]string{})
	if err != nil {
		z.sendError(c, EventNetProtocol, err)
		return
	}
	z.sendEvent(c, EventNetProtocol, stats)
}

func (z *Client) headEventMonitorInfo(c app_control.Control) {
	var userUserName, userDisplayName, userUid string
	if usr, err := user.Current(); err == nil {
		userUserName = usr.Username
		userDisplayName = usr.Name
		userUid = usr.Uid
	}

	z.sendEvent(c, EventMonitorInfo, struct {
		AppVersion      string `json:"app_version"`
		MonitorName     string `json:"monitor_name"`
		IntervalMonitor int    `json:"interval_monitor"`
		IntervalSync    int    `json:"interval_sync"`
		UserDisplayName string `json:"user_display_name"`
		UserUid         string `json:"user_uid"`
		UserName        string `json:"user_name"`
	}{
		AppVersion:      app.BuildId,
		MonitorName:     z.Name,
		IntervalMonitor: z.MonitorInterval.Value(),
		IntervalSync:    z.SyncInterval.Value(),
		UserDisplayName: userDisplayName,
		UserUid:         userUid,
		UserName:        userUserName,
	})
}

func (z *Client) headEvents(c app_control.Control) {
	z.headEventMonitorInfo(c)
	z.headEventCpuInfo(c)
	z.headEventHostInfo(c)
	z.headEventDiskPartition(c)
}

func (z *Client) Exec(c app_control.Control) error {
	l := c.Log()
	if err := os.MkdirAll(z.DataPath.Path(), 0755); err != nil {
		return err
	}
	if err := z.openJournal(c); err != nil {
		return err
	}

	// Head events
	z.headEvents(c)

	// Periodical events
	for {
		z.eventCpuPercent(c)
		z.eventCpuTime(c)
		z.eventDiskUsage(c)
		z.eventLoadAverage(c)
		z.eventMemoryStat(c)
		z.eventNetIO(c)
		z.eventNetProtocol(c)

		if !z.MonitorEnd.IsZero() && z.MonitorEnd.Time().Before(time.Now()) {
			return z.syncJournal(c)
		}
		time.Sleep(time.Duration(z.MonitorInterval.Value()) * time.Second)
		l.Info("Monitor", esl.Time("t", time.Now()))
	}
}

func (z *Client) Test(c app_control.Control) error {
	f, err := qt_file.MakeTestFolder("monitor", false)
	if err != nil {
		return err
	}
	defer func() {
		_ = os.RemoveAll(f)
	}()

	return rc_exec.ExecMock(c, &Client{}, func(r rc_recipe.Recipe) {
		m := r.(*Client)
		m.Name = "mango"
		m.DataPath = mo_path.NewFileSystemPath(f)
		m.SyncPath = qtr_endtoend.NewTestDropboxFolderPath("monitor")
		m.MonitorEnd = mo_time.NewOptional(time.Now())
	})
}
<commit_msg>fix #650 : head events in every files<commit_after>package monitor

import (
	"encoding/json"
	"fmt"
	"github.com/shirou/gopsutil/v3/cpu"
	"github.com/shirou/gopsutil/v3/disk"
	"github.com/shirou/gopsutil/v3/host"
	"github.com/shirou/gopsutil/v3/load"
	"github.com/shirou/gopsutil/v3/mem"
	"github.com/shirou/gopsutil/v3/net"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_auth"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn"
	mo_path2 "github.com/watermint/toolbox/domain/dropbox/model/mo_path"
	"github.com/watermint/toolbox/domain/dropbox/model/mo_time"
	"github.com/watermint/toolbox/domain/dropbox/service/sv_file_content"
	"github.com/watermint/toolbox/essentials/file/es_filepath"
	"github.com/watermint/toolbox/essentials/file/es_gzip"
	"github.com/watermint/toolbox/essentials/log/esl"
	"github.com/watermint/toolbox/essentials/model/mo_int"
	"github.com/watermint/toolbox/essentials/model/mo_path"
	"github.com/watermint/toolbox/infra/app"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/quality/infra/qt_file"
	"github.com/watermint/toolbox/quality/recipe/qtr_endtoend"
	"os"
	"os/user"
	"path/filepath"
	"strings"
	"time"
)

const (
	monitorFilePrefix = "tbx-monitor-"
)

type Client struct {
	DataPath        mo_path.FileSystemPath
	SyncPath        mo_path2.DropboxPath
	Peer            dbx_conn.ConnScopedIndividual
	Name            string
	MonitorInterval mo_int.RangeInt
	SyncInterval    mo_int.RangeInt
	MonitorEnd      mo_time.TimeOptional
	Display         bool

	sentErrors       map[string]int64 // eventType -> num error sent
	sentCount        map[string]int64 // eventType -> num event sent
	currentJournal   *os.File
	currentPath      string
	currentStart     time.Time
	currentDeadline  time.Time
	rotateInProgress bool
}

func (z *Client) Preset() {
	z.MonitorInterval.SetRange(1, 86400, 10)
	z.SyncInterval.SetRange(10, 86400, 3600)
	z.sentErrors = make(map[string]int64)
	z.sentCount = make(map[string]int64)
	z.Peer.SetScopes(
		dbx_auth.ScopeFilesContentWrite,
	)
}
func (z *Client) sendError(c app_control.Control, eventType string, err error) {
	if n, ok := z.sentErrors[eventType]; ok {
		z.sentErrors[eventType] = n + 1
		return
	}
	l := c.Log()
	l.Warn("Unable to retrieve event data", esl.String("type", eventType), esl.Error(err))
	z.sentErrors[eventType] = 1
}

func (z *Client) openJournal(c app_control.Control) error {
	l := c.Log()
	if z.currentJournal != nil {
		return nil
	}
	name := monitorFilePrefix + es_filepath.Escape(z.Name) + "-" + fmt.Sprintf("%08x", time.Now().Unix()) + ".log"
	path := filepath.Join(z.DataPath.Path(), name)
	l = l.With(esl.String("path", path))
	f, err := os.Create(path)
	if err != nil {
		l.Debug("Unable to create the log file", esl.Error(err))
		return err
	}
	z.currentJournal = f
	z.currentPath = path
	z.currentStart = time.Now()
	z.currentDeadline = z.currentStart.Add(time.Duration(z.SyncInterval.Value()) * time.Second)
	l.Debug("Journal created", esl.Time("deadline", z.currentDeadline))
	return nil
}

func (z *Client) syncJournal(c app_control.Control) error {
	l := c.Log()
	if err := z.currentJournal.Close(); err != nil {
		l.Debug("Unable to close", esl.Error(err))
	}
	if _, err := es_gzip.Compress(z.currentPath); err != nil {
		l.Debug("Unable to compress", esl.Error(err))
		return err
	}
	z.currentJournal = nil
	z.currentPath = ""

	files, err := os.ReadDir(z.DataPath.Path())
	if err != nil {
		l.Debug("Unable to read directory entry")
		return err
	}

	sv := sv_file_content.NewUpload(z.Peer.Client())
	basePath := z.SyncPath.ChildPath(es_filepath.Escape(z.Name), z.currentStart.Format("2006-01"), z.currentStart.Format("2006-01-02"))
	for _, f := range files {
		if !strings.HasPrefix(f.Name(), monitorFilePrefix) {
			continue
		}
		fp := filepath.Join(z.DataPath.Path(), f.Name())
		l.Info("Syncing journal file", esl.String("name", f.Name()))
		entry, err := sv.Add(basePath, fp)
		if err != nil {
			l.Debug("Unable to upload", esl.Error(err))
			return err
		}
		l.Debug("Upload completed", esl.Any("entry", entry))
		_ = os.Remove(fp)
	}
	return nil
}

func (z *Client) sendEvent(c app_control.Control, eventType string, data interface{}) {
	ev := Event{
		Time: time.Now().Format(time.RFC3339),
		Type: eventType,
		Data: data,
	}
	evs, err := json.Marshal(&ev)
	if err != nil {
		z.sendError(c, eventType, err)
		return
	}
	if z.Display {
		c.Log().Info("event", esl.Any("data", ev))
	}

	if z.currentJournal != nil {
		_, err0 := z.currentJournal.Write(evs)
		_, err1 := z.currentJournal.Write([]byte("\n"))

		if z.rotateInProgress {
			return
		}

		if err0 == nil && err1 == nil && z.currentDeadline.Before(time.Now()) {
			z.rotateInProgress = true
			if err := z.syncJournal(c); err != nil {
				z.sendError(c, eventType, err)
			}
			if err := z.openJournal(c); err != nil {
				z.sendError(c, eventType, err)
			}
			z.headEvents(c)
			z.rotateInProgress = false
		}
	}

	if ns, ok := z.sentCount[eventType]; ok {
		z.sentCount[eventType] = ns + 1
	} else {
		z.sentCount[eventType] = 1
	}
}

func (z *Client) shouldAbort(eventType string) bool {
	if ne, oke := z.sentErrors[eventType]; oke {
		if ns, okc := z.sentCount[eventType]; okc {
			// Abort when three more errors without success
			if 2 < ne && ns < 1 {
				return true
			}
		}
	}
	return false
}

func (z *Client) handleEvent(c app_control.Control, eventType string, f func() (data interface{}, err error)) {
	if z.shouldAbort(eventType) {
		return
	}

	data, err := f()
	if err != nil {
		z.sendError(c, eventType, err)
	} else {
		z.sendEvent(c, eventType, data)
	}
}

func (z *Client) headEventCpuInfo(c app_control.Control) {
	z.handleEvent(c, EventCpuInfo, func() (data interface{}, err error) {
		return cpu.Info()
	})
}

func (z *Client) eventCpuTime(c app_control.Control) {
	z.handleEvent(c, EventCpuTime, func() (data interface{}, err error) {
		return cpu.Times(true)
	})
}

func (z *Client) eventCpuPercent(c app_control.Control) {
	z.handleEvent(c, EventCpuPercent, func() (data interface{}, err error) {
		return cpu.Percent(0, true)
	})
}

func (z *Client) headEventHostInfo(c app_control.Control) {
	z.handleEvent(c, EventHostInfo, func() (data interface{}, err error) {
		return host.Info()
	})
}

func (z *Client) headEventDiskPartition(c app_control.Control) {
	z.handleEvent(c, EventDiskPartition, func() (data interface{}, err error) {
		return disk.Partitions(true)
	})
}

func (z *Client) eventDiskUsage(c app_control.Control) {
	if z.shouldAbort(EventDiskUsage) {
		return
	}

	partitions, err := disk.Partitions(true)
	if err != nil {
		z.sendError(c, EventDiskUsage, err)
		return
	}
	for _, p := range partitions {
		usage, err := disk.Usage(p.Mountpoint)
		if err != nil {
			z.sendError(c, EventDiskUsage, err)
			continue
		}
		z.sendEvent(c, EventDiskUsage, usage)
	}
}

func (z *Client) eventLoadAverage(c app_control.Control) {
	z.handleEvent(c, EventLoadAverage, func() (data interface{}, err error) {
		return load.Avg()
	})
}

func (z *Client) eventMemoryStat(c app_control.Control) {
	z.handleEvent(c, EventMemoryStat, func() (data interface{}, err error) {
		return mem.VirtualMemory()
	})
}

func (z *Client) eventNetIO(c app_control.Control) {
	z.handleEvent(c, EventNetIO, func() (data interface{}, err error) {
		return net.IOCounters(true)
	})
}

func (z *Client) eventNetProtocol(c app_control.Control) {
	z.handleEvent(c, EventNetProtocol, func() (data interface{}, err error) {
		return net.ProtoCounters([]string{})
	})
}

func (z *Client) headEventMonitorInfo(c app_control.Control) {
	var userUserName, userDisplayName, userUid string
	if usr, err := user.Current(); err == nil {
		userUserName = usr.Username
		userDisplayName = usr.Name
		userUid = usr.Uid
	}

	z.sendEvent(c, EventMonitorInfo, struct {
		AppVersion      string `json:"app_version"`
		MonitorName     string `json:"monitor_name"`
		IntervalMonitor int    `json:"interval_monitor"`
		IntervalSync    int    `json:"interval_sync"`
		UserDisplayName string `json:"user_display_name"`
		UserUid         string `json:"user_uid"`
		UserName        string `json:"user_name"`
	}{
		AppVersion:      app.BuildId,
		MonitorName:     z.Name,
		IntervalMonitor: z.MonitorInterval.Value(),
		IntervalSync:    z.SyncInterval.Value(),
		UserDisplayName: userDisplayName,
		UserUid:         userUid,
		UserName:        userUserName,
	})
}

func (z *Client) headEvents(c app_control.Control) {
	z.headEventMonitorInfo(c)
	z.headEventCpuInfo(c)
	z.headEventHostInfo(c)
	z.headEventDiskPartition(c)
}

func (z *Client) Exec(c app_control.Control) error {
	l := c.Log()
	if err := os.MkdirAll(z.DataPath.Path(), 0755); err != nil {
		return err
	}
	if err := z.openJournal(c); err != nil {
		return err
	}

	// Head events
	z.headEvents(c)

	// Periodical events
	for {
		z.eventCpuPercent(c)
		z.eventCpuTime(c)
		z.eventDiskUsage(c)
		z.eventLoadAverage(c)
		z.eventMemoryStat(c)
		z.eventNetIO(c)
		z.eventNetProtocol(c)

		if !z.MonitorEnd.IsZero() && z.MonitorEnd.Time().Before(time.Now()) {
			return z.syncJournal(c)
		}
		time.Sleep(time.Duration(z.MonitorInterval.Value()) * time.Second)
		l.Info("Monitor", esl.Time("t", time.Now()))
	}
}

func (z *Client) Test(c app_control.Control) error {
	f, err := qt_file.MakeTestFolder("monitor", false)
	if err != nil {
		return err
	}
	defer func() {
		_ = os.RemoveAll(f)
	}()

	return rc_exec.ExecMock(c, &Client{}, func(r rc_recipe.Recipe) {
		m := r.(*Client)
		m.Name = "mango"
		m.DataPath = mo_path.NewFileSystemPath(f)
		m.SyncPath = qtr_endtoend.NewTestDropboxFolderPath("monitor")
		m.MonitorEnd = mo_time.NewOptional(time.Now())
	})
}
<|endoftext|>


<commit_before>// Copyright 2017 The Grafeas Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package testutil

import (
	"fmt"
	"log"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/any"
	pb "github.com/grafeas/grafeas/v1alpha1/proto"
	opspb "google.golang.org/genproto/googleapis/longrunning"
)

func Occurrence(pID, noteName string) *pb.Occurrence {
	return &pb.Occurrence{
		Name:        fmt.Sprintf("projects/%s/occurrences/134", pID),
		ResourceUrl: "gcr.io/foo/bar",
		NoteName:    noteName,
		Kind:        pb.Note_PACKAGE_VULNERABILITY,
		Details: &pb.Occurrence_VulnerabilityDetails{
			VulnerabilityDetails: &pb.VulnerabilityType_VulnerabilityDetails{
				Severity:  pb.VulnerabilityType_HIGH,
				CvssScore: 7.5,
				PackageIssue: []*pb.VulnerabilityType_PackageIssue{
					&pb.VulnerabilityType_PackageIssue{
						SeverityName: "HIGH",
						AffectedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:8",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "52.1",
								Revision: "8+deb8u3",
							},
						},
						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:8",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "52.1",
								Revision: "8+deb8u4",
							},
						},
					},
				},
			},
		},
	}
}

func Note(pID string) *pb.Note {
	return &pb.Note{
		Name:             fmt.Sprintf("projects/%s/notes/CVE-1999-0710", pID),
		ShortDescription: "CVE-2014-9911",
		LongDescription:  "NIST vectors: AV:N/AC:L/Au:N/C:P/I:P",
		Kind:             pb.Note_PACKAGE_VULNERABILITY,
		NoteType: &pb.Note_VulnerabilityType{
			&pb.VulnerabilityType{
				CvssScore: 7.5,
				Severity:  pb.VulnerabilityType_HIGH,
				Details: []*pb.VulnerabilityType_Detail{
					&pb.VulnerabilityType_Detail{
						CpeUri:  "cpe:/o:debian:debian_linux:7",
						Package: "icu",
						Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " +
							"common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " +
							"remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.",
						MinAffectedVersion: &pb.VulnerabilityType_Version{
							Kind: pb.VulnerabilityType_Version_MINIMUM,
						},
						SeverityName: "HIGH",

						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:7",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "4.8.1.1",
								Revision: "12+deb7u6",
							},
						},
					},
					&pb.VulnerabilityType_Detail{
						CpeUri:  "cpe:/o:debian:debian_linux:8",
						Package: "icu",
						Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " +
							"common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " +
							"remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.",
						MinAffectedVersion: &pb.VulnerabilityType_Version{
							Kind: pb.VulnerabilityType_Version_MINIMUM,
						},
						SeverityName: "HIGH",

						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:8",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "52.1",
								Revision: "8+deb8u4",
							},
						},
					},
					&pb.VulnerabilityType_Detail{
						CpeUri:  "cpe:/o:debian:debian_linux:9",
						Package: "icu",
						Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " +
							"common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " +
							"remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.",
						MinAffectedVersion: &pb.VulnerabilityType_Version{
							Kind: pb.VulnerabilityType_Version_MINIMUM,
						},
						SeverityName: "HIGH",

						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:9",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "55.1",
								Revision: "3",
							},
						},
					},
					&pb.VulnerabilityType_Detail{
						CpeUri:  "cpe:/o:canonical:ubuntu_linux:14.04",
						Package: "andriod",
						Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " +
							"common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " +
							"remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.",
						MinAffectedVersion: &pb.VulnerabilityType_Version{
							Kind: pb.VulnerabilityType_Version_MINIMUM,
						},
						SeverityName: "MEDIUM",

						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:canonical:ubuntu_linux:14.04",
							Package: "andriod",
							Version: &pb.VulnerabilityType_Version{
								Kind: pb.VulnerabilityType_Version_MAXIMUM,
							},
						},
					},
				},
			},
		},
		RelatedUrl: []*pb.Note_RelatedUrl{
			&pb.Note_RelatedUrl{
				Url:   "https://security-tracker.debian.org/tracker/CVE-2014-9911",
				Label: "More Info",
			},
			&pb.Note_RelatedUrl{
				Url:   "http://people.ubuntu.com/~ubuntu-security/cve/CVE-2014-9911",
				Label: "More Info",
			},
		},
	}
}

func Operation(pID string) *opspb.Operation {
	md := &pb.OperationMetadata{CreateTime: ptypes.TimestampNow()}
	bytes, err := proto.Marshal(md)
	if err != nil {
		log.Printf("Error parsing bytes: %v", err)
		return nil
	}
	return &opspb.Operation{
		Name:     fmt.Sprintf("projects/%s/operations/foo", pID),
		Metadata: &any.Any{Value: bytes},
		Done:     false,
	}
}
<commit_msg>Fix misspelling of 'android'<commit_after>// Copyright 2017 The Grafeas Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package testutil

import (
	"fmt"
	"log"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/any"
	pb "github.com/grafeas/grafeas/v1alpha1/proto"
	opspb "google.golang.org/genproto/googleapis/longrunning"
)

func Occurrence(pID, noteName string) *pb.Occurrence {
	return &pb.Occurrence{
		Name:        fmt.Sprintf("projects/%s/occurrences/134", pID),
		ResourceUrl: "gcr.io/foo/bar",
		NoteName:    noteName,
		Kind:        pb.Note_PACKAGE_VULNERABILITY,
		Details: &pb.Occurrence_VulnerabilityDetails{
			VulnerabilityDetails: &pb.VulnerabilityType_VulnerabilityDetails{
				Severity:  pb.VulnerabilityType_HIGH,
				CvssScore: 7.5,
				PackageIssue: []*pb.VulnerabilityType_PackageIssue{
					&pb.VulnerabilityType_PackageIssue{
						SeverityName: "HIGH",
						AffectedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:8",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "52.1",
								Revision: "8+deb8u3",
							},
						},
						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:8",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "52.1",
								Revision: "8+deb8u4",
							},
						},
					},
				},
			},
		},
	}
}

func Note(pID string) *pb.Note {
	return &pb.Note{
		Name:             fmt.Sprintf("projects/%s/notes/CVE-1999-0710", pID),
		ShortDescription: "CVE-2014-9911",
		LongDescription:  "NIST vectors: AV:N/AC:L/Au:N/C:P/I:P",
		Kind:             pb.Note_PACKAGE_VULNERABILITY,
		NoteType: &pb.Note_VulnerabilityType{
			&pb.VulnerabilityType{
				CvssScore: 7.5,
				Severity:  pb.VulnerabilityType_HIGH,
				Details: []*pb.VulnerabilityType_Detail{
					&pb.VulnerabilityType_Detail{
						CpeUri:  "cpe:/o:debian:debian_linux:7",
						Package: "icu",
						Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " +
							"common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " +
							"remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.",
						MinAffectedVersion: &pb.VulnerabilityType_Version{
							Kind: pb.VulnerabilityType_Version_MINIMUM,
						},
						SeverityName: "HIGH",

						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:7",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "4.8.1.1",
								Revision: "12+deb7u6",
							},
						},
					},
					&pb.VulnerabilityType_Detail{
						CpeUri:  "cpe:/o:debian:debian_linux:8",
						Package: "icu",
						Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " +
							"common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " +
							"remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.",
						MinAffectedVersion: &pb.VulnerabilityType_Version{
							Kind: pb.VulnerabilityType_Version_MINIMUM,
						},
						SeverityName: "HIGH",

						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:8",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "52.1",
								Revision: "8+deb8u4",
							},
						},
					},
					&pb.VulnerabilityType_Detail{
						CpeUri:  "cpe:/o:debian:debian_linux:9",
						Package: "icu",
						Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " +
							"common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " +
							"remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.",
						MinAffectedVersion: &pb.VulnerabilityType_Version{
							Kind: pb.VulnerabilityType_Version_MINIMUM,
						},
						SeverityName: "HIGH",

						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:debian:debian_linux:9",
							Package: "icu",
							Version: &pb.VulnerabilityType_Version{
								Name:     "55.1",
								Revision: "3",
							},
						},
					},
					&pb.VulnerabilityType_Detail{
						CpeUri:  "cpe:/o:canonical:ubuntu_linux:14.04",
						Package: "android",
						Description: "Stack-based buffer overflow in the ures_getByKeyWithFallback function in " +
							"common/uresbund.cpp in International Components for Unicode (ICU) before 54.1 for C/C++ allows " +
							"remote attackers to cause a denial of service or possibly have unspecified other impact via a crafted uloc_getDisplayName call.",
						MinAffectedVersion: &pb.VulnerabilityType_Version{
							Kind: pb.VulnerabilityType_Version_MINIMUM,
						},
						SeverityName: "MEDIUM",

						FixedLocation: &pb.VulnerabilityType_VulnerabilityLocation{
							CpeUri:  "cpe:/o:canonical:ubuntu_linux:14.04",
							Package: "android",
							Version: &pb.VulnerabilityType_Version{
								Kind: pb.VulnerabilityType_Version_MAXIMUM,
							},
						},
					},
				},
			},
		},
		RelatedUrl: []*pb.Note_RelatedUrl{
			&pb.Note_RelatedUrl{
				Url:   "https://security-tracker.debian.org/tracker/CVE-2014-9911",
				Label: "More Info",
			},
			&pb.Note_RelatedUrl{
				Url:   "http://people.ubuntu.com/~ubuntu-security/cve/CVE-2014-9911",
				Label: "More Info",
			},
		},
	}
}

func Operation(pID string) *opspb.Operation {
	md := &pb.OperationMetadata{CreateTime: ptypes.TimestampNow()}
	bytes, err := proto.Marshal(md)
	if err != nil {
		log.Printf("Error parsing bytes: %v", err)
		return nil
	}
	return &opspb.Operation{
		Name:     fmt.Sprintf("projects/%s/operations/foo", pID),
		Metadata: &any.Any{Value: bytes},
		Done:     false,
	}
}
<|endoftext|>


<commit_before>package replication

import (
	"errors"
	"time"
)

var (
	ErrGetEventTimeout = errors.New("Get event timeout, try get later")
	ErrNeedSyncAgain   = errors.New("Last sync error or closed, try sync and get event again")
	ErrSyncClosed      = errors.New("Sync was closed")
)

type BinlogStreamer struct {
	ch  chan *BinlogEvent
	ech chan error
	err error
}

func (s *BinlogStreamer) GetEvent() (*BinlogEvent, error) {
	// we use a very very long timeout here
	return s.GetEventTimeout(time.Second * 3600 * 24 * 30)
}

// if timeout, ErrGetEventTimeout will returns
func (s *BinlogStreamer) GetEventTimeout(d time.Duration) (*BinlogEvent, error) {
	if s.err != nil {
		return nil, ErrNeedSyncAgain
	}

	select {
	case c := <-s.ch:
		return c, nil
	case s.err = <-s.ech:
		return nil, s.err
	case <-time.After(d):
		return nil, ErrGetEventTimeout
	}
}

func (s *BinlogStreamer) close() {
	s.closeWithError(ErrSyncClosed)
}

func (s *BinlogStreamer) closeWithError(err error) {
	if err == nil {
		err = ErrSyncClosed
	}
	select {
	case s.ech <- err:
	default:
	}
}

func newBinlogStreamer() *BinlogStreamer {
	s := new(BinlogStreamer)

	s.ch = make(chan *BinlogEvent, 1024)
	s.ech = make(chan error, 4)

	return s
}
<commit_msg>set timeout to 1s to avoid memory leak<commit_after>package replication

import (
	"errors"
	"time"
)

var (
	ErrGetEventTimeout = errors.New("Get event timeout, try get later")
	ErrNeedSyncAgain   = errors.New("Last sync error or closed, try sync and get event again")
	ErrSyncClosed      = errors.New("Sync was closed")
)

type BinlogStreamer struct {
	ch  chan *BinlogEvent
	ech chan error
	err error
}

func (s *BinlogStreamer) GetEvent() (*BinlogEvent, error) {
	// we use a very very long timeout here
	return s.GetEventTimeout(time.Second * 1)
}

// if timeout, ErrGetEventTimeout will returns
func (s *BinlogStreamer) GetEventTimeout(d time.Duration) (*BinlogEvent, error) {
	if s.err != nil {
		return nil, ErrNeedSyncAgain
	}

	select {
	case c := <-s.ch:
		return c, nil
	case s.err = <-s.ech:
		return nil, s.err
	case <-time.After(d):
		return nil, ErrGetEventTimeout
	}
}

func (s *BinlogStreamer) close() {
	s.closeWithError(ErrSyncClosed)
}

func (s *BinlogStreamer) closeWithError(err error) {
	if err == nil {
		err = ErrSyncClosed
	}
	select {
	case s.ech <- err:
	default:
	}
}

func newBinlogStreamer() *BinlogStreamer {
	s := new(BinlogStreamer)

	s.ch = make(chan *BinlogEvent, 1024)
	s.ech = make(chan error, 4)

	return s
}
<|endoftext|>


<commit_before>package content

import (
	"encoding/json"
	"regexp"
	"time"

	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// CypherDriver - CypherDriver
type service struct {
	conn neoutils.NeoConnection
}

//NewCypherDriver instantiate driver
func NewCypherContentService(cypherRunner neoutils.NeoConnection) service {
	return service{cypherRunner}
}

//Initialise ensure constraint on content uuid
func (cd service) Initialise() error {

	return cd.conn.EnsureConstraints(map[string]string{
		"Content": "uuid"})
}

// Check - Feeds into the Healthcheck and checks whether we can connect to Neo and that the datastore isn't empty
func (pcd service) Check() error {
	return neoutils.Check(pcd.conn)
}

// Read - reads a content given a UUID
func (pcd service) Read(uuid string) (interface{}, bool, error) {
	results := []struct {
		content
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Content {uuid:{uuid}})
			return n.uuid as uuid, n.title as title, n.publishedDate as publishedDate`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &results,
	}

	err := pcd.conn.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return content{}, false, err
	}

	if len(results) == 0 {
		return content{}, false, nil
	}

	result := results[0]

	contentItem := content{
		UUID:          result.UUID,
		Title:         result.Title,
		PublishedDate: result.PublishedDate,
	}
	return contentItem, true, nil
}

//Write - Writes a content node
func (pcd service) Write(thing interface{}) error {
	c := thing.(content)

	// Only Articles have a body
	if c.Body == "" {
		log.Infof("There is no body with this content item therefore assuming is it not an Article: %v", c.UUID)
		return nil
	}

	params := map[string]interface{}{
		"uuid": c.UUID,
	}

	if c.Title != "" {
		params["title"] = c.Title
		params["prefLabel"] = c.Title
	}

	if c.PublishedDate != "" {
		params["publishedDate"] = c.PublishedDate
		datetimeEpoch, err := time.Parse(time.RFC3339, c.PublishedDate)

		if err != nil {
			return err
		}

		params["publishedDateEpoch"] = datetimeEpoch.Unix()
	}

	statement := `MERGE (n:Thing {uuid: {uuid}})
		      set n={allprops}
		      set n :Content`

	return pcd.conn.CypherBatch(
		[]*neoism.CypherQuery{
			{
				Statement: statement,
				Parameters: map[string]interface{}{
					"uuid":     c.UUID,
					"allprops": params,
				},
			},
		},
	)
}

//Delete - Deletes a content
func (pcd service) Delete(uuid string) (bool, error) {
	clearNode := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			REMOVE p:Content
			SET p={props}
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
			"props": map[string]interface{}{
				"uuid": uuid,
			},
		},
		IncludeStats: true,
	}

	removeNodeIfUnused := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			OPTIONAL MATCH (p)-[a]-(x)
			WITH p, count(a) AS relCount
			WHERE relCount = 0
			DELETE p
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
	}

	err := pcd.conn.CypherBatch([]*neoism.CypherQuery{clearNode, removeNodeIfUnused})

	s1, err := clearNode.Stats()
	if err != nil {
		return false, err
	}

	var deleted bool
	if s1.ContainsUpdates && s1.LabelsRemoved > 0 {
		deleted = true
	}

	return deleted, err
}

// DecodeJSON - Decodes JSON into content
func (pcd service) DecodeJSON(dec *json.Decoder) (interface{}, string, error) {
	c := content{}
	err := dec.Decode(&c)
	return c, c.UUID, err

}

// Count - Returns a count of the number of content in this Neo instance
func (pcd service) Count() (int, error) {

	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Content) return count(n) as c`,
		Result:    &results,
	}

	err := pcd.conn.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

<commit_msg>Removed inline in return statement<commit_after>package content

import (
	"encoding/json"
	"regexp"
	"time"

	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// CypherDriver - CypherDriver
type service struct {
	conn neoutils.NeoConnection
}

//NewCypherDriver instantiate driver
func NewCypherContentService(cypherRunner neoutils.NeoConnection) service {
	return service{cypherRunner}
}

//Initialise ensure constraint on content uuid
func (cd service) Initialise() error {

	return cd.conn.EnsureConstraints(map[string]string{
		"Content": "uuid"})
}

// Check - Feeds into the Healthcheck and checks whether we can connect to Neo and that the datastore isn't empty
func (pcd service) Check() error {
	return neoutils.Check(pcd.conn)
}

// Read - reads a content given a UUID
func (pcd service) Read(uuid string) (interface{}, bool, error) {
	results := []struct {
		content
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Content {uuid:{uuid}})
			return n.uuid as uuid, n.title as title, n.publishedDate as publishedDate`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &results,
	}

	err := pcd.conn.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return content{}, false, err
	}

	if len(results) == 0 {
		return content{}, false, nil
	}

	result := results[0]

	contentItem := content{
		UUID:          result.UUID,
		Title:         result.Title,
		PublishedDate: result.PublishedDate,
	}
	return contentItem, true, nil
}

//Write - Writes a content node
func (pcd service) Write(thing interface{}) error {
	c := thing.(content)

	// Only Articles have a body
	if c.Body == "" {
		log.Infof("There is no body with this content item therefore assuming is it not an Article: %v", c.UUID)
		return nil
	}

	params := map[string]interface{}{
		"uuid": c.UUID,
	}

	if c.Title != "" {
		params["title"] = c.Title
		params["prefLabel"] = c.Title
	}

	if c.PublishedDate != "" {
		params["publishedDate"] = c.PublishedDate
		datetimeEpoch, err := time.Parse(time.RFC3339, c.PublishedDate)

		if err != nil {
			return err
		}

		params["publishedDateEpoch"] = datetimeEpoch.Unix()
	}

	statement := `MERGE (n:Thing {uuid: {uuid}})
		      set n={allprops}
		      set n :Content`

	writeContentQuery := &neoism.CypherQuery{
		Statement: statement,
		Parameters: map[string]interface{}{
			"uuid":     c.UUID,
			"allprops": params,
		},
	}

	return pcd.conn.CypherBatch([]*neoism.CypherQuery{writeContentQuery})
}

//Delete - Deletes a content
func (pcd service) Delete(uuid string) (bool, error) {
	clearNode := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			REMOVE p:Content
			SET p={props}
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
			"props": map[string]interface{}{
				"uuid": uuid,
			},
		},
		IncludeStats: true,
	}

	removeNodeIfUnused := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			OPTIONAL MATCH (p)-[a]-(x)
			WITH p, count(a) AS relCount
			WHERE relCount = 0
			DELETE p
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
	}

	err := pcd.conn.CypherBatch([]*neoism.CypherQuery{clearNode, removeNodeIfUnused})

	s1, err := clearNode.Stats()
	if err != nil {
		return false, err
	}

	var deleted bool
	if s1.ContainsUpdates && s1.LabelsRemoved > 0 {
		deleted = true
	}

	return deleted, err
}

// DecodeJSON - Decodes JSON into content
func (pcd service) DecodeJSON(dec *json.Decoder) (interface{}, string, error) {
	c := content{}
	err := dec.Decode(&c)
	return c, c.UUID, err

}

// Count - Returns a count of the number of content in this Neo instance
func (pcd service) Count() (int, error) {

	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Content) return count(n) as c`,
		Result:    &results,
	}

	err := pcd.conn.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

<|endoftext|>


<commit_before>package marathon

import (
	"fmt"
	//"log"
	"time"

	"github.com/Banno/go-marathon"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"

	"testing"
)

const testAccCheckMarathonAppConfig_basic = `
resource "marathon_app" "app-create-example" {
	name = "/app-create-example"

	cmd = "env && python3 -m http.server $PORT0"

	container {
		type = "DOCKER"
		docker {
			image = "python:3"
                }
	}

	cpus = "0.01"
	instances = 1
	mem = 100

}
`

const testAccCheckMarathonAppConfig_update = `
resource "marathon_app" "app-create-example" {
	name = "/app-create-example"

	cmd = "env && python3 -m http.server $PORT0"

	container {
		type = "DOCKER"
		docker {
			image = "python:3"
                }
	}

	cpus = "0.01"
	instances = 2
	mem = 100

}
`

func TestAccMarathonApp_basic(t *testing.T) {

	var a marathon.App

	testCheckCreate := func(app *marathon.App) resource.TestCheckFunc {
		return func(s *terraform.State) error {
			if a.Version == "" {
				return fmt.Errorf("Didn't return a version so something is broken: %#v", app)
			}
			if a.Instances != 1 {
				return fmt.Errorf("Wrong number of instances %#v", app)
			}
			return nil
		}
	}

	testCheckUpdate := func(app *marathon.App) resource.TestCheckFunc {
		return func(s *terraform.State) error {
			if a.Instances != 2 {
				return fmt.Errorf("Wrong number of instances %#v", app)

			}
			return nil
		}
	}

	resource.Test(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		//		CheckDestroy: testAccCheckMarathonAppDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccCheckMarathonAppConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccReadApp("marathon_app.app-create-example", &a),
					testCheckCreate(&a),
				),
			},
			resource.TestStep{
				Config: testAccCheckMarathonAppConfig_update,
				Check: resource.ComposeTestCheckFunc(
					testAccReadApp("marathon_app.app-create-example", &a),
					testCheckUpdate(&a),
				),
			},
		},
	})
}

func testAccReadApp(name string, app *marathon.App) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("marathon_app resource not found: %s", name)
		}
		if rs.Primary.ID == "" {
			return fmt.Errorf("marathon_app resource id not set correctly: %s", name)
		}

		//log.Printf("=== testAccContainerExists: rs ===\n%#v\n", rs)

		client := testAccProvider.Meta().(*marathon.Client)

		appRead, _ := client.AppRead(rs.Primary.Attributes["name"])

		//		log.Printf("=== testAccContainerExists: appRead ===\n%#v\n", appRead)

		time.Sleep(5000 * time.Millisecond)

		*app = *appRead

		return nil
	}
}

/*
TODO: prove that this works

func testAccCheckMarathonAppDestroy(s *terraform.State) error {
	client := testAccProvider.Meta().(*marathon.Client)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "marathon_app" {
			continue
		}

		client.AppRead("/test")

		// make sure that it's properly destroyed
	}

	return nil
}
*/
<commit_msg>Confirm deletion test passes now<commit_after>package marathon

import (
	"fmt"
	"time"

	"github.com/Banno/go-marathon"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"

	"testing"
)

const testAccCheckMarathonAppConfig_basic = `
resource "marathon_app" "app-create-example" {
	name = "/app-create-example"

	cmd = "env && python3 -m http.server $PORT0"

	container {
		type = "DOCKER"
		docker {
			image = "python:3"
                }
	}

	cpus = "0.01"
	instances = 1
	mem = 100

}
`

const testAccCheckMarathonAppConfig_update = `
resource "marathon_app" "app-create-example" {
	name = "/app-create-example"

	cmd = "env && python3 -m http.server $PORT0"

	container {
		type = "DOCKER"
		docker {
			image = "python:3"
                }
	}

	cpus = "0.01"
	instances = 2
	mem = 100

}
`

func TestAccMarathonApp_basic(t *testing.T) {

	var a marathon.App

	testCheckCreate := func(app *marathon.App) resource.TestCheckFunc {
		return func(s *terraform.State) error {
			time.Sleep(1 * time.Second)
			if a.Version == "" {
				return fmt.Errorf("Didn't return a version so something is broken: %#v", app)
			}
			if a.Instances != 1 {
				return fmt.Errorf("AppCreate: Wrong number of instances %#v", app)
			}
			return nil
		}
	}

	testCheckUpdate := func(app *marathon.App) resource.TestCheckFunc {
		return func(s *terraform.State) error {
			if a.Instances != 2 {
				return fmt.Errorf("AppUpdate: Wrong number of instances %#v", app)

			}
			return nil
		}
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckMarathonAppDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccCheckMarathonAppConfig_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccReadApp("marathon_app.app-create-example", &a),
					testCheckCreate(&a),
				),
			},
			resource.TestStep{
				Config: testAccCheckMarathonAppConfig_update,
				Check: resource.ComposeTestCheckFunc(
					testAccReadApp("marathon_app.app-create-example", &a),
					testCheckUpdate(&a),
				),
			},
		},
	})
}

func testAccReadApp(name string, app *marathon.App) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[name]
		if !ok {
			return fmt.Errorf("marathon_app resource not found: %s", name)
		}
		if rs.Primary.ID == "" {
			return fmt.Errorf("marathon_app resource id not set correctly: %s", name)
		}

		//log.Printf("=== testAccContainerExists: rs ===\n%#v\n", rs)

		client := testAccProvider.Meta().(*marathon.Client)

		appRead, _ := client.AppRead(rs.Primary.Attributes["name"])

		//		log.Printf("=== testAccContainerExists: appRead ===\n%#v\n", appRead)

		time.Sleep(5000 * time.Millisecond)

		*app = *appRead

		return nil
	}
}

func testAccCheckMarathonAppDestroy(s *terraform.State) error {

	client := testAccProvider.Meta().(*marathon.Client)

	_, err := client.AppRead("/app-create-example")
	if err == nil {
		return fmt.Errorf("App not deleted! %#v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"strings"
	"text/template"

	"github.com/docker/docker/api/types"
	flag "github.com/docker/docker/pkg/mflag"
)

// CmdInspect displays low-level information on one or more containers or images.
//
// Usage: docker inspect [OPTIONS] CONTAINER|IMAGE [CONTAINER|IMAGE...]

func (cli *DockerCli) CmdInspect(args ...string) error {
	cmd := cli.Subcmd("inspect", "CONTAINER|IMAGE [CONTAINER|IMAGE...]", "Return low-level information on a container or image", true)
	tmplStr := cmd.String([]string{"f", "#format", "-format"}, "", "Format the output using the given go template")
	cmd.Require(flag.Min, 1)

	cmd.ParseFlags(args, true)

	var tmpl *template.Template
	if *tmplStr != "" {
		var err error
		if tmpl, err = template.New("").Funcs(funcMap).Parse(*tmplStr); err != nil {
			fmt.Fprintf(cli.err, "Template parsing error: %v\n", err)
			return StatusError{StatusCode: 64,
				Status: "Template parsing error: " + err.Error()}
		}
	}

	indented := new(bytes.Buffer)
	indented.WriteByte('[')
	status := 0
	isImage := false

	for _, name := range cmd.Args() {
		obj, _, err := readBody(cli.call("GET", "/containers/"+name+"/json", nil, nil))
		if err != nil {
			obj, _, err = readBody(cli.call("GET", "/images/"+name+"/json", nil, nil))
			isImage = true
			if err != nil {
				if strings.Contains(err.Error(), "No such") {
					fmt.Fprintf(cli.err, "Error: No such image or container: %s\n", name)
				} else {
					fmt.Fprintf(cli.err, "%s", err)
				}
				status = 1
				continue
			}
		}

		if tmpl == nil {
			if err = json.Indent(indented, obj, "", "    "); err != nil {
				fmt.Fprintf(cli.err, "%s\n", err)
				status = 1
				continue
			}
		} else {
			dec := json.NewDecoder(bytes.NewReader(obj))

			if isImage {
				inspPtr := types.ImageInspect{}
				if err := dec.Decode(&inspPtr); err != nil {
					fmt.Fprintf(cli.err, "%s\n", err)
					status = 1
					continue
				}
				if err := tmpl.Execute(cli.out, inspPtr); err != nil {
					return err
				}
			} else {
				inspPtr := types.ContainerJSON{}
				if err := dec.Decode(&inspPtr); err != nil {
					fmt.Fprintf(cli.err, "%s\n", err)
					status = 1
					continue
				}
				if err := tmpl.Execute(cli.out, inspPtr); err != nil {
					return err

				}
			}
			cli.out.Write([]byte{'\n'})
		}
		indented.WriteString(",")
	}

	if indented.Len() > 1 {
		// Remove trailing ','
		indented.Truncate(indented.Len() - 1)
	}
	indented.WriteString("]\n")

	if tmpl == nil {
		if _, err := io.Copy(cli.out, indented); err != nil {
			return err
		}
	}

	if status != 0 {
		return StatusError{StatusCode: status}
	}
	return nil
}
<commit_msg>fix a minor inspect format issue<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"strings"
	"text/template"

	"github.com/docker/docker/api/types"
	flag "github.com/docker/docker/pkg/mflag"
)

// CmdInspect displays low-level information on one or more containers or images.
//
// Usage: docker inspect [OPTIONS] CONTAINER|IMAGE [CONTAINER|IMAGE...]

func (cli *DockerCli) CmdInspect(args ...string) error {
	cmd := cli.Subcmd("inspect", "CONTAINER|IMAGE [CONTAINER|IMAGE...]", "Return low-level information on a container or image", true)
	tmplStr := cmd.String([]string{"f", "#format", "-format"}, "", "Format the output using the given go template")
	cmd.Require(flag.Min, 1)

	cmd.ParseFlags(args, true)

	var tmpl *template.Template
	if *tmplStr != "" {
		var err error
		if tmpl, err = template.New("").Funcs(funcMap).Parse(*tmplStr); err != nil {
			fmt.Fprintf(cli.err, "Template parsing error: %v\n", err)
			return StatusError{StatusCode: 64,
				Status: "Template parsing error: " + err.Error()}
		}
	}

	indented := new(bytes.Buffer)
	indented.WriteString("[\n")
	status := 0
	isImage := false

	for _, name := range cmd.Args() {
		obj, _, err := readBody(cli.call("GET", "/containers/"+name+"/json", nil, nil))
		if err != nil {
			obj, _, err = readBody(cli.call("GET", "/images/"+name+"/json", nil, nil))
			isImage = true
			if err != nil {
				if strings.Contains(err.Error(), "No such") {
					fmt.Fprintf(cli.err, "Error: No such image or container: %s\n", name)
				} else {
					fmt.Fprintf(cli.err, "%s", err)
				}
				status = 1
				continue
			}
		}

		if tmpl == nil {
			if err = json.Indent(indented, obj, "", "    "); err != nil {
				fmt.Fprintf(cli.err, "%s\n", err)
				status = 1
				continue
			}
		} else {
			dec := json.NewDecoder(bytes.NewReader(obj))

			if isImage {
				inspPtr := types.ImageInspect{}
				if err := dec.Decode(&inspPtr); err != nil {
					fmt.Fprintf(cli.err, "%s\n", err)
					status = 1
					continue
				}
				if err := tmpl.Execute(cli.out, inspPtr); err != nil {
					return err
				}
			} else {
				inspPtr := types.ContainerJSON{}
				if err := dec.Decode(&inspPtr); err != nil {
					fmt.Fprintf(cli.err, "%s\n", err)
					status = 1
					continue
				}
				if err := tmpl.Execute(cli.out, inspPtr); err != nil {
					return err

				}
			}
			cli.out.Write([]byte{'\n'})
		}
		indented.WriteString(",")
	}

	if indented.Len() > 1 {
		// Remove trailing ','
		indented.Truncate(indented.Len() - 1)
	}
	indented.WriteString("]\n")

	if tmpl == nil {
		if _, err := io.Copy(cli.out, indented); err != nil {
			return err
		}
	}

	if status != 0 {
		return StatusError{StatusCode: status}
	}
	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"
	"time"
)

const (
	maxRetryDuration = 5 * time.Minute
)

// Request is contructed iteratively by the client and finally dispatched.
// A REST endpoint is accessed with the following convention:
// base_url/<version>/<resource>/[<instance>]
type Request struct {
	client      *http.Client
	version     string
	verb        string
	path        string
	base        *url.URL
	params      url.Values
	headers     http.Header
	resource    string
	instance    string
	err         error
	body        []byte
	req         *http.Request
	resp        *http.Response
	timeout     time.Duration
	authstring  string
	accesstoken string
}

// Response is a representation of HTTP response received from the server.
type Response struct {
	status     string
	statusCode int
	err        error
	body       []byte
}

// Status upon error, attempts to parse the body of a response into a meaningful status.
type Status struct {
	Message   string
	ErrorCode int
}

// NewRequest instance
func NewRequest(client *http.Client, base *url.URL, verb string, version string, authstring, userAgent string) *Request {
	r := &Request{
		client:     client,
		verb:       verb,
		base:       base,
		path:       base.Path,
		version:    version,
		authstring: authstring,
	}
	r.SetHeader("User-Agent", userAgent)
	return r
}

func checkExists(mustExist string, before string) error {
	if len(mustExist) == 0 {
		return fmt.Errorf("%q should be set before setting %q", mustExist, before)
	}
	return nil
}

func checkSet(name string, s *string, newval string) error {
	if len(*s) != 0 {
		return fmt.Errorf("%q already set to %q, cannot change to %q",
			name, *s, newval)
	}
	*s = newval
	return nil
}

// Resource specifies the resource to be accessed.
func (r *Request) Resource(resource string) *Request {
	if r.err == nil {
		r.err = checkSet("resource", &r.resource, resource)
	}
	return r
}

// Instance specifies the instance of the resource to be accessed.
func (r *Request) Instance(instance string) *Request {
	if r.err == nil {
		r.err = checkExists("resource", "instance")
		if r.err == nil {
			r.err = checkSet("instance", &r.instance, instance)
		}
	}
	return r
}

// UsePath use the specified path and don't build up a request.
func (r *Request) UsePath(path string) *Request {
	if r.err == nil {
		r.err = checkSet("path", &r.path, path)
	}
	return r
}

// QueryOption adds specified options to query.
func (r *Request) QueryOption(key string, value string) *Request {
	if r.err != nil {
		return r
	}
	if r.params == nil {
		r.params = make(url.Values)
	}
	r.params.Add(string(key), value)
	return r
}

// QueryOptionLabel adds specified label to query.
func (r *Request) QueryOptionLabel(key string, labels map[string]string) *Request {
	if r.err != nil {
		return r
	}
	if b, err := json.Marshal(labels); err != nil {
		r.err = err
	} else {
		if r.params == nil {
			r.params = make(url.Values)
		}
		r.params.Add(string(key), string(b))
	}
	return r
}

// SetHeader adds specified header values to query.
func (r *Request) SetHeader(key, value string) *Request {
	if r.headers == nil {
		r.headers = http.Header{}
	}
	r.headers.Set(key, value)
	return r
}

// Timeout makes the request use the given duration as a timeout. Sets the "timeout"
// parameter.
func (r *Request) Timeout(d time.Duration) *Request {
	if r.err != nil {
		return r
	}
	r.timeout = d
	return r
}

// Body sets the request Body.
func (r *Request) Body(v interface{}) *Request {
	var err error
	if r.err != nil {
		return r
	}
	r.body, err = json.Marshal(v)
	if err != nil {
		r.err = err
		return r
	}
	return r
}

// URL returns the current working URL.
func (r *Request) URL() *url.URL {
	u := *r.base
	p := r.path

	if len(r.version) != 0 {
		p = path.Join(p, strings.ToLower(r.version))
	}
	if len(r.resource) != 0 {
		p = path.Join(p, strings.ToLower(r.resource))
		if len(r.instance) != 0 {
			p = path.Join(p, r.instance)
		}
	}

	u.Path = p

	query := url.Values{}
	for key, values := range r.params {
		for _, value := range values {
			query.Add(key, value)
		}
	}
	if r.timeout != 0 {
		query.Set("timeout", r.timeout.String())
	}
	u.RawQuery = query.Encode()
	return &u
}

// headerVal for key as an int. Return false if header is not present or valid.
func headerVal(key string, resp *http.Response) (int, bool) {
	if h := resp.Header.Get(key); len(h) > 0 {
		if i, err := strconv.Atoi(h); err == nil {
			return i, true
		}
	}
	return 0, false
}

func parseHTTPStatus(resp *http.Response, body []byte) error {
	if resp.StatusCode >= http.StatusOK &&
		resp.StatusCode <= http.StatusPartialContent {
		// Status is good and HTTP status is good, everything is good
		return nil
	}

	// Get error from body if any
	if len(string(body)) != 0 {
		return errors.New(string(body))
	}

	// If no error was in the body, return a generic one
	return fmt.Errorf("HTTP error %d", resp.StatusCode)
}

// Do executes the request and returns a Response.
// Do executes the request and returns a Response.
func (r *Request) Do() *Response {
	var (
		err  error
		req  *http.Request
		resp *http.Response
		url  string
		body []byte
	)

	if r.err != nil {
		return &Response{err: r.err}
	}
	url = r.URL().String()
	req, err = http.NewRequest(r.verb, url, bytes.NewBuffer(r.body))
	if err != nil {
		return &Response{err: err}
	}
	if r.headers == nil {
		r.headers = http.Header{}
	}

	req.Header = r.headers
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Date", time.Now().String())

	if len(r.authstring) > 0 {
		req.Header.Set("Authorization", "Basic "+r.authstring)
	}

	if len(r.accesstoken) > 0 {
		req.Header.Set("Access-Token", r.accesstoken)
	}

	start := time.Now()
	for {
		if resp, err = r.client.Do(req); err != nil {
			return &Response{err: err}
		}

		if time.Since(start) >= maxRetryDuration ||
			resp.StatusCode != http.StatusServiceUnavailable {
			// Server needs to set this header along with returning a 503
			break
		}
		handleServiceUnavailable(resp)
	}

	if resp.Body != nil {
		defer resp.Body.Close()
		if body, err = ioutil.ReadAll(resp.Body); err != nil {
			return &Response{err: err}
		}
	}

	return &Response{
		status:     resp.Status,
		statusCode: resp.StatusCode,
		body:       body,
		err:        parseHTTPStatus(resp, body),
	}
}

func handleServiceUnavailable(resp *http.Response) {
	var duration = time.Duration(1 * time.Second)
	if len(resp.Header["Retry-After"]) > 0 {
		if retryafter, err := strconv.Atoi(resp.Header["Retry-After"][0]); err == nil {
			duration = time.Duration(retryafter) * time.Second
		}
	}

	time.Sleep(duration)
}

// Body return http body, valid only if there is no error
func (r Response) Body() ([]byte, error) {
	return r.body, r.err
}

// StatusCode HTTP status code returned.
func (r Response) StatusCode() int {
	return r.statusCode
}

// Unmarshal result into obj
func (r Response) Unmarshal(v interface{}) error {
	if r.err != nil {
		return r.err
	}
	return json.Unmarshal(r.body, v)
}

// Error executing the request.
func (r Response) Error() error {
	return r.err
}

// FormatError formats the error
func (r Response) FormatError() error {
	if len(r.body) == 0 {
		return fmt.Errorf("Error: %v", r.err)
	}
	return fmt.Errorf("HTTP-%d: %s", r.statusCode, string(r.body))
}

func digest(method string, path string) string {
	now := time.Now().String()

	s1 := rand.NewSource(time.Now().UnixNano())
	r1 := rand.New(s1)

	nonce := r1.Intn(10)

	return method + "+" + path + "+" + now + "+" + strconv.Itoa(nonce)
}
<commit_msg>Don't print HTTP status in error message<commit_after>package client

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"path"
	"strconv"
	"strings"
	"time"
)

const (
	maxRetryDuration = 5 * time.Minute
)

// Request is contructed iteratively by the client and finally dispatched.
// A REST endpoint is accessed with the following convention:
// base_url/<version>/<resource>/[<instance>]
type Request struct {
	client      *http.Client
	version     string
	verb        string
	path        string
	base        *url.URL
	params      url.Values
	headers     http.Header
	resource    string
	instance    string
	err         error
	body        []byte
	req         *http.Request
	resp        *http.Response
	timeout     time.Duration
	authstring  string
	accesstoken string
}

// Response is a representation of HTTP response received from the server.
type Response struct {
	status     string
	statusCode int
	err        error
	body       []byte
}

// Status upon error, attempts to parse the body of a response into a meaningful status.
type Status struct {
	Message   string
	ErrorCode int
}

// NewRequest instance
func NewRequest(client *http.Client, base *url.URL, verb string, version string, authstring, userAgent string) *Request {
	r := &Request{
		client:     client,
		verb:       verb,
		base:       base,
		path:       base.Path,
		version:    version,
		authstring: authstring,
	}
	r.SetHeader("User-Agent", userAgent)
	return r
}

func checkExists(mustExist string, before string) error {
	if len(mustExist) == 0 {
		return fmt.Errorf("%q should be set before setting %q", mustExist, before)
	}
	return nil
}

func checkSet(name string, s *string, newval string) error {
	if len(*s) != 0 {
		return fmt.Errorf("%q already set to %q, cannot change to %q",
			name, *s, newval)
	}
	*s = newval
	return nil
}

// Resource specifies the resource to be accessed.
func (r *Request) Resource(resource string) *Request {
	if r.err == nil {
		r.err = checkSet("resource", &r.resource, resource)
	}
	return r
}

// Instance specifies the instance of the resource to be accessed.
func (r *Request) Instance(instance string) *Request {
	if r.err == nil {
		r.err = checkExists("resource", "instance")
		if r.err == nil {
			r.err = checkSet("instance", &r.instance, instance)
		}
	}
	return r
}

// UsePath use the specified path and don't build up a request.
func (r *Request) UsePath(path string) *Request {
	if r.err == nil {
		r.err = checkSet("path", &r.path, path)
	}
	return r
}

// QueryOption adds specified options to query.
func (r *Request) QueryOption(key string, value string) *Request {
	if r.err != nil {
		return r
	}
	if r.params == nil {
		r.params = make(url.Values)
	}
	r.params.Add(string(key), value)
	return r
}

// QueryOptionLabel adds specified label to query.
func (r *Request) QueryOptionLabel(key string, labels map[string]string) *Request {
	if r.err != nil {
		return r
	}
	if b, err := json.Marshal(labels); err != nil {
		r.err = err
	} else {
		if r.params == nil {
			r.params = make(url.Values)
		}
		r.params.Add(string(key), string(b))
	}
	return r
}

// SetHeader adds specified header values to query.
func (r *Request) SetHeader(key, value string) *Request {
	if r.headers == nil {
		r.headers = http.Header{}
	}
	r.headers.Set(key, value)
	return r
}

// Timeout makes the request use the given duration as a timeout. Sets the "timeout"
// parameter.
func (r *Request) Timeout(d time.Duration) *Request {
	if r.err != nil {
		return r
	}
	r.timeout = d
	return r
}

// Body sets the request Body.
func (r *Request) Body(v interface{}) *Request {
	var err error
	if r.err != nil {
		return r
	}
	r.body, err = json.Marshal(v)
	if err != nil {
		r.err = err
		return r
	}
	return r
}

// URL returns the current working URL.
func (r *Request) URL() *url.URL {
	u := *r.base
	p := r.path

	if len(r.version) != 0 {
		p = path.Join(p, strings.ToLower(r.version))
	}
	if len(r.resource) != 0 {
		p = path.Join(p, strings.ToLower(r.resource))
		if len(r.instance) != 0 {
			p = path.Join(p, r.instance)
		}
	}

	u.Path = p

	query := url.Values{}
	for key, values := range r.params {
		for _, value := range values {
			query.Add(key, value)
		}
	}
	if r.timeout != 0 {
		query.Set("timeout", r.timeout.String())
	}
	u.RawQuery = query.Encode()
	return &u
}

// headerVal for key as an int. Return false if header is not present or valid.
func headerVal(key string, resp *http.Response) (int, bool) {
	if h := resp.Header.Get(key); len(h) > 0 {
		if i, err := strconv.Atoi(h); err == nil {
			return i, true
		}
	}
	return 0, false
}

func parseHTTPStatus(resp *http.Response, body []byte) error {
	if resp.StatusCode >= http.StatusOK &&
		resp.StatusCode <= http.StatusPartialContent {
		// Status is good and HTTP status is good, everything is good
		return nil
	}

	// Get error from body if any
	if len(string(body)) != 0 {
		return errors.New(string(body))
	}

	// If no error was in the body, return a generic one
	return fmt.Errorf("HTTP error %d", resp.StatusCode)
}

// Do executes the request and returns a Response.
// Do executes the request and returns a Response.
func (r *Request) Do() *Response {
	var (
		err  error
		req  *http.Request
		resp *http.Response
		url  string
		body []byte
	)

	if r.err != nil {
		return &Response{err: r.err}
	}
	url = r.URL().String()
	req, err = http.NewRequest(r.verb, url, bytes.NewBuffer(r.body))
	if err != nil {
		return &Response{err: err}
	}
	if r.headers == nil {
		r.headers = http.Header{}
	}

	req.Header = r.headers
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Date", time.Now().String())

	if len(r.authstring) > 0 {
		req.Header.Set("Authorization", "Basic "+r.authstring)
	}

	if len(r.accesstoken) > 0 {
		req.Header.Set("Access-Token", r.accesstoken)
	}

	start := time.Now()
	for {
		if resp, err = r.client.Do(req); err != nil {
			return &Response{err: err}
		}

		if time.Since(start) >= maxRetryDuration ||
			resp.StatusCode != http.StatusServiceUnavailable {
			// Server needs to set this header along with returning a 503
			break
		}
		handleServiceUnavailable(resp)
	}

	if resp.Body != nil {
		defer resp.Body.Close()
		if body, err = ioutil.ReadAll(resp.Body); err != nil {
			return &Response{err: err}
		}
	}

	return &Response{
		status:     resp.Status,
		statusCode: resp.StatusCode,
		body:       body,
		err:        parseHTTPStatus(resp, body),
	}
}

func handleServiceUnavailable(resp *http.Response) {
	var duration = time.Duration(1 * time.Second)
	if len(resp.Header["Retry-After"]) > 0 {
		if retryafter, err := strconv.Atoi(resp.Header["Retry-After"][0]); err == nil {
			duration = time.Duration(retryafter) * time.Second
		}
	}

	time.Sleep(duration)
}

// Body return http body, valid only if there is no error
func (r Response) Body() ([]byte, error) {
	return r.body, r.err
}

// StatusCode HTTP status code returned.
func (r Response) StatusCode() int {
	return r.statusCode
}

// Unmarshal result into obj
func (r Response) Unmarshal(v interface{}) error {
	if r.err != nil {
		return r.err
	}
	return json.Unmarshal(r.body, v)
}

// Error executing the request.
func (r Response) Error() error {
	return r.err
}

// FormatError formats the error
func (r Response) FormatError() error {
	if len(r.body) == 0 {
		return fmt.Errorf("Error: %v", r.err)
	}
	return fmt.Errorf("%v", strings.TrimSpace(string(r.body)))
}

func digest(method string, path string) string {
	now := time.Now().String()

	s1 := rand.NewSource(time.Now().UnixNano())
	r1 := rand.New(s1)

	nonce := r1.Intn(10)

	return method + "+" + path + "+" + now + "+" + strconv.Itoa(nonce)
}
<|endoftext|>


<commit_before>// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package api

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/git-lfs/git-lfs/auth"
	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/httputil"
)

var (
	// ErrNoOperationGiven is an error which is returned when no operation
	// is provided in a RequestSchema object.
	ErrNoOperationGiven = errors.New("lfs/api: no operation provided in schema")
)

// HttpLifecycle serves as the default implementation of the Lifecycle interface
// for HTTP requests. Internally, it leverages the *http.Client type to execute
// HTTP requests against a root *url.URL, as given in `NewHttpLifecycle`.
type HttpLifecycle struct {
	cfg *config.Configuration
}

var _ Lifecycle = new(HttpLifecycle)

// NewHttpLifecycle initializes a new instance of the *HttpLifecycle type with a
// new *http.Client, and the given root (see above).
// Passing a nil Configuration will use the global config
func NewHttpLifecycle(cfg *config.Configuration) *HttpLifecycle {
	if cfg == nil {
		cfg = config.Config
	}
	return &HttpLifecycle{
		cfg: cfg,
	}
}

// Build implements the Lifecycle.Build function.
//
// HttpLifecycle in particular, builds an absolute path by parsing and then
// relativizing the `schema.Path` with respsect to the `HttpLifecycle.root`. If
// there was an error in determining this URL, then that error will be returned,
//
// After this is complete, a body is attached to the request if the
// schema contained one. If a body was present, and there an error occurred while
// serializing it into JSON, then that error will be returned and the
// *http.Request will not be generated.
//
// In all cases, credentials are attached to the HTTP request as described in
// the `auth` package (see github.com/git-lfs/git-lfs/auth#GetCreds).
//
// Finally, all of these components are combined together and the resulting
// request is returned.
func (l *HttpLifecycle) Build(schema *RequestSchema) (*http.Request, error) {
	path, err := l.absolutePath(schema.Operation, schema.Path)
	if err != nil {
		return nil, err
	}

	body, err := l.body(schema)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest(schema.Method, path.String(), body)
	if err != nil {
		return nil, err
	}

	if _, err = auth.GetCreds(l.cfg, req); err != nil {
		return nil, err
	}

	req.URL.RawQuery = l.queryParameters(schema).Encode()

	return req, nil
}

// Execute implements the Lifecycle.Execute function.
//
// Internally, the *http.Client is used to execute the underlying *http.Request.
// If the client returned an error corresponding to a failure to make the
// request, then that error will be returned immediately, and the response is
// guaranteed not to be serialized.
//
// Once the response has been gathered from the server, it is unmarshled into
// the given `into interface{}` which is identical to the one provided in the
// original RequestSchema. If an error occured while decoding, then that error
// is returned.
//
// Otherwise, the api.Response is returned, along with no error, signaling that
// the request completed successfully.
func (l *HttpLifecycle) Execute(req *http.Request, into interface{}) (Response, error) {
	resp, err := httputil.DoHttpRequestWithRedirects(l.cfg, req, []*http.Request{}, true)
	if err != nil {
		return nil, err
	}

	// TODO(taylor): check status >=500, handle content type, return error,
	// halt immediately.

	if into != nil {
		decoder := json.NewDecoder(resp.Body)
		if err = decoder.Decode(into); err != nil {
			return nil, err
		}
	}

	return WrapHttpResponse(resp), nil
}

// Cleanup implements the Lifecycle.Cleanup function by closing the Body
// attached to the response.
func (l *HttpLifecycle) Cleanup(resp Response) error {
	return resp.Body().Close()
}

// absolutePath returns the absolute path made by combining a given relative
// path with the root URL of the endpoint corresponding to the given operation.
//
// If there was an error in parsing the relative path, then that error will be
// returned.
func (l *HttpLifecycle) absolutePath(operation Operation, path string) (*url.URL, error) {
	if len(operation) == 0 {
		return nil, ErrNoOperationGiven
	}

	root, err := url.Parse(l.cfg.Endpoint(string(operation)).Url)
	if err != nil {
		return nil, err
	}

	rel, err := url.Parse(path)
	if err != nil {
		return nil, err

	}

	return root.ResolveReference(rel), nil
}

// body returns an io.Reader which reads out a JSON-encoded copy of the payload
// attached to a given *RequestSchema, if it is present. If no body is present
// in the request, then nil is returned instead.
//
// If an error was encountered while attempting to marshal the body, then that
// will be returned instead, along with a nil io.Reader.
func (l *HttpLifecycle) body(schema *RequestSchema) (io.ReadCloser, error) {
	if schema.Body == nil {
		return nil, nil
	}

	body, err := json.Marshal(schema.Body)
	if err != nil {
		return nil, err
	}

	return ioutil.NopCloser(bytes.NewReader(body)), nil
}

// queryParameters returns a url.Values containing all of the provided query
// parameters as given in the *RequestSchema. If no query parameters were given,
// then an empty url.Values is returned instead.
func (l *HttpLifecycle) queryParameters(schema *RequestSchema) url.Values {
	vals := url.Values{}
	if schema.Query != nil {
		for k, v := range schema.Query {
			vals.Add(k, v)
		}
	}

	return vals
}
<commit_msg>api: simpler path join<commit_after>// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package api

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"

	"github.com/git-lfs/git-lfs/auth"
	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/httputil"
)

var (
	// ErrNoOperationGiven is an error which is returned when no operation
	// is provided in a RequestSchema object.
	ErrNoOperationGiven = errors.New("lfs/api: no operation provided in schema")
)

// HttpLifecycle serves as the default implementation of the Lifecycle interface
// for HTTP requests. Internally, it leverages the *http.Client type to execute
// HTTP requests against a root *url.URL, as given in `NewHttpLifecycle`.
type HttpLifecycle struct {
	cfg *config.Configuration
}

var _ Lifecycle = new(HttpLifecycle)

// NewHttpLifecycle initializes a new instance of the *HttpLifecycle type with a
// new *http.Client, and the given root (see above).
// Passing a nil Configuration will use the global config
func NewHttpLifecycle(cfg *config.Configuration) *HttpLifecycle {
	if cfg == nil {
		cfg = config.Config
	}
	return &HttpLifecycle{
		cfg: cfg,
	}
}

// Build implements the Lifecycle.Build function.
//
// HttpLifecycle in particular, builds an absolute path by parsing and then
// relativizing the `schema.Path` with respsect to the `HttpLifecycle.root`. If
// there was an error in determining this URL, then that error will be returned,
//
// After this is complete, a body is attached to the request if the
// schema contained one. If a body was present, and there an error occurred while
// serializing it into JSON, then that error will be returned and the
// *http.Request will not be generated.
//
// In all cases, credentials are attached to the HTTP request as described in
// the `auth` package (see github.com/git-lfs/git-lfs/auth#GetCreds).
//
// Finally, all of these components are combined together and the resulting
// request is returned.
func (l *HttpLifecycle) Build(schema *RequestSchema) (*http.Request, error) {
	path, err := l.absolutePath(schema.Operation, schema.Path)
	if err != nil {
		return nil, err
	}

	body, err := l.body(schema)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest(schema.Method, path.String(), body)
	if err != nil {
		return nil, err
	}

	if _, err = auth.GetCreds(l.cfg, req); err != nil {
		return nil, err
	}

	req.URL.RawQuery = l.queryParameters(schema).Encode()

	return req, nil
}

// Execute implements the Lifecycle.Execute function.
//
// Internally, the *http.Client is used to execute the underlying *http.Request.
// If the client returned an error corresponding to a failure to make the
// request, then that error will be returned immediately, and the response is
// guaranteed not to be serialized.
//
// Once the response has been gathered from the server, it is unmarshled into
// the given `into interface{}` which is identical to the one provided in the
// original RequestSchema. If an error occured while decoding, then that error
// is returned.
//
// Otherwise, the api.Response is returned, along with no error, signaling that
// the request completed successfully.
func (l *HttpLifecycle) Execute(req *http.Request, into interface{}) (Response, error) {
	resp, err := httputil.DoHttpRequestWithRedirects(l.cfg, req, []*http.Request{}, true)
	if err != nil {
		return nil, err
	}

	// TODO(taylor): check status >=500, handle content type, return error,
	// halt immediately.

	if into != nil {
		decoder := json.NewDecoder(resp.Body)
		if err = decoder.Decode(into); err != nil {
			return nil, err
		}
	}

	return WrapHttpResponse(resp), nil
}

// Cleanup implements the Lifecycle.Cleanup function by closing the Body
// attached to the response.
func (l *HttpLifecycle) Cleanup(resp Response) error {
	return resp.Body().Close()
}

// absolutePath returns the absolute path made by combining a given relative
// path with the root URL of the endpoint corresponding to the given operation.
//
// If there was an error in parsing the relative path, then that error will be
// returned.
func (l *HttpLifecycle) absolutePath(operation Operation, relpath string) (*url.URL, error) {
	if len(operation) == 0 {
		return nil, ErrNoOperationGiven
	}

	root, err := url.Parse(l.cfg.Endpoint(string(operation)).Url)
	if err != nil {
		return nil, err
	}

	root.Path = path.Join(root.Path, relpath)
	return root, nil
}

// body returns an io.Reader which reads out a JSON-encoded copy of the payload
// attached to a given *RequestSchema, if it is present. If no body is present
// in the request, then nil is returned instead.
//
// If an error was encountered while attempting to marshal the body, then that
// will be returned instead, along with a nil io.Reader.
func (l *HttpLifecycle) body(schema *RequestSchema) (io.ReadCloser, error) {
	if schema.Body == nil {
		return nil, nil
	}

	body, err := json.Marshal(schema.Body)
	if err != nil {
		return nil, err
	}

	return ioutil.NopCloser(bytes.NewReader(body)), nil
}

// queryParameters returns a url.Values containing all of the provided query
// parameters as given in the *RequestSchema. If no query parameters were given,
// then an empty url.Values is returned instead.
func (l *HttpLifecycle) queryParameters(schema *RequestSchema) url.Values {
	vals := url.Values{}
	if schema.Query != nil {
		for k, v := range schema.Query {
			vals.Add(k, v)
		}
	}

	return vals
}
<|endoftext|>


<commit_before>/* Data functions */

/*
 * Copyright (c) 2013, Jeremy Bingham (<jbingham@gmail.com>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"net/http"
	"encoding/json"
	"fmt"
	"github.com/ctdk/goiardi/data_bag"
	"github.com/ctdk/goiardi/util"
)

func data_handler(w http.ResponseWriter, r *http.Request){
	w.Header().Set("Content-Type", "application/json")

	path_array := SplitPath(r.URL.Path)

	db_response := make(map[string]interface{})
	if len(path_array) == 1 {
		/* Either a list of data bags, or a POST to create a new one */
		switch r.Method {
			case "GET":
				/* The list */
				db_list := data_bag.GetList()
				for _, k := range db_list {
					item_url := fmt.Sprintf("/data/%s", k)
					db_response[k] = util.CustomURL(item_url)
				}
			case "POST":
				db_data, jerr := ParseObjJson(r.Body)
				if jerr != nil {
					JsonErrorReport(w, r, jerr.Error(), http.StatusBadRequest)
					return
				}
				/* check that the name exists */
				switch t := db_data["name"].(type) {
					case string:
						if t == "" {
							JsonErrorReport(w, r, "Field 'name' missing", http.StatusBadRequest)
							return
						}
					default:
						JsonErrorReport(w, r, "Field 'name' missing", http.StatusBadRequest)
						return
				}
				chef_dbag, _ := data_bag.Get(db_data["name"].(string))
				if chef_dbag != nil {
					httperr := fmt.Errorf("Data bag %s already exists.", db_data["name"].(string))
					JsonErrorReport(w, r, httperr.Error(), http.StatusConflict)
					return
				}
				chef_dbag, nerr := data_bag.New(db_data["name"].(string))
				if nerr != nil {
					JsonErrorReport(w, r, nerr.Error(), nerr.Status())
					return
				}
				chef_dbag.Save()
				db_response["uri"] = util.ObjURL(chef_dbag)
				w.WriteHeader(http.StatusCreated)
			default:
				/* The chef-pedant spec wants this response for
				 * some reason. Mix it up, I guess. */
				JsonErrorReport(w, r, "GET, PUT", http.StatusMethodNotAllowed)
				return
		}
	} else {
		db_name := path_array[1]

		/* chef-pedant is unhappy about not reporting the HTTP status
		 * as 404 by fetching the data bag before we see if the method
		 * is allowed, so do a quick check for that here. */
		if (len(path_array) == 2  && r.Method == "PUT") || (len(path_array) == 3 && r.Method == "POST"){
			JsonErrorReport(w, r, "Method not allowed", http.StatusMethodNotAllowed)
			return
		}
		chef_dbag, err := data_bag.Get(db_name)
		if err != nil {
			var err_msg string
			status := err.Status()
			if r.Method == "POST" {
				/* Posts get a special snowflake message */
				err_msg = fmt.Sprintf("No data bag '%s' could be found. Please create this data bag before adding items to it.", db_name)
			} else {
				if len(path_array) == 3 {
					/* This is nuts. */
					if r.Method == "DELETE" {
						err_msg = fmt.Sprintf("Cannot load data bag %s item %s", db_name, path_array[2])
					} else {
						err_msg = fmt.Sprintf("Cannot load data bag item %s for data bag %s", path_array[2], db_name)
					}
				} else {
					err_msg = err.Error()
				}
			}
			JsonErrorReport(w, r, err_msg, status)
			return
		}
		if len(path_array) == 2 {
			/* getting list of data bag items and creating data bag
			 * items. */
			switch r.Method {
				case "GET":
					for k, _ := range chef_dbag.DataBagItems {
						db_response[k] = util.CustomObjURL(chef_dbag, k)
					}
				case "DELETE":
					/* The chef API docs don't say anything
					 * about this existing, but it does,
					 * and without it you can't delete data
					 * bags at all. */
					db_response["chef_type"] = "data_bag"
					db_response["json_class"] = "Chef::DataBag"
					db_response["name"] = chef_dbag.Name
					err := chef_dbag.Delete()
					if err != nil {
						JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
						return
					}
				case "POST":
					raw_data := data_bag.RawDataBagJson(r.Body)
					dbitem, nerr := chef_dbag.NewDBItem(raw_data)
					if nerr != nil {
						JsonErrorReport(w, r, nerr.Error(), nerr.Status())
						return
					}

					/* The data bag return values are all
					 * kinds of weird. Sometimes it sends
					 * just the raw data, sometimes it sends
					 * the whole object, sometimes a special
					 * snowflake version. Ugh. */
					db_response = dbitem.RawData
					//db_response["data_bag"] = dbitem.DataBagName
					//db_response["chef_type"] = dbitem.ChefType
					w.WriteHeader(http.StatusCreated)
				default:
					JsonErrorReport(w, r, "GET, DELETE, POST", http.StatusMethodNotAllowed)
					return
			}
		} else {
			/* getting, editing, and deleting existing data bag items. */
			db_item_name := path_array[2]
			if _, ok := chef_dbag.DataBagItems[db_item_name]; !ok {
				var httperr string
				if r.Method != "DELETE" {
					httperr = fmt.Sprintf("Cannot load data bag item %s for data bag %s", db_item_name, chef_dbag.Name)
				} else {
					httperr = fmt.Sprintf("Cannot load data bag %s item %s", chef_dbag.Name, db_item_name)
				}
				JsonErrorReport(w, r, httperr, http.StatusNotFound)
				return
			}
			switch r.Method {
				case "GET":
					db_response = chef_dbag.DataBagItems[db_item_name].RawData
				case "DELETE":
					dbi := chef_dbag.DataBagItems[db_item_name]
					/* Gotta short circuit this */
					enc := json.NewEncoder(w)
					if err := enc.Encode(&dbi); err != nil {
						JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
						return
					}
					err := chef_dbag.DeleteDBItem(db_item_name)
					if err != nil {
						JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
						return
					}
					return
				case "PUT":
					raw_data := data_bag.RawDataBagJson(r.Body)
					dbitem, err := chef_dbag.UpdateDBItem(db_item_name, raw_data)
					if err != nil {
						JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
						return
					}
					/* Another weird data bag item response
					 * which isn't at all unusual. */
					db_response = dbitem.RawData
					db_response["data_bag"] = dbitem.DataBagName
					db_response["chef_type"] = dbitem.ChefType
					db_response["id"] = db_item_name
				default:
					JsonErrorReport(w, r, "GET, DELETE, PUT", http.StatusMethodNotAllowed)
					return
			}
		}
	}

	enc := json.NewEncoder(w)
	if err := enc.Encode(&db_response); err != nil {
		JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
	}
}

<commit_msg>Data Bag tests now passing except for perms, max allowed size payloads still being touchy, and some really picky 'method not allowed' tests.<commit_after>/* Data functions */

/*
 * Copyright (c) 2013, Jeremy Bingham (<jbingham@gmail.com>)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"net/http"
	"encoding/json"
	"fmt"
	"github.com/ctdk/goiardi/data_bag"
	"github.com/ctdk/goiardi/util"
)

func data_handler(w http.ResponseWriter, r *http.Request){
	w.Header().Set("Content-Type", "application/json")

	path_array := SplitPath(r.URL.Path)

	db_response := make(map[string]interface{})
	if len(path_array) == 1 {
		/* Either a list of data bags, or a POST to create a new one */
		switch r.Method {
			case "GET":
				/* The list */
				db_list := data_bag.GetList()
				for _, k := range db_list {
					item_url := fmt.Sprintf("/data/%s", k)
					db_response[k] = util.CustomURL(item_url)
				}
			case "POST":
				db_data, jerr := ParseObjJson(r.Body)
				if jerr != nil {
					JsonErrorReport(w, r, jerr.Error(), http.StatusBadRequest)
					return
				}
				/* check that the name exists */
				switch t := db_data["name"].(type) {
					case string:
						if t == "" {
							JsonErrorReport(w, r, "Field 'name' missing", http.StatusBadRequest)
							return
						}
					default:
						JsonErrorReport(w, r, "Field 'name' missing", http.StatusBadRequest)
						return
				}
				chef_dbag, _ := data_bag.Get(db_data["name"].(string))
				if chef_dbag != nil {
					httperr := fmt.Errorf("Data bag %s already exists.", db_data["name"].(string))
					JsonErrorReport(w, r, httperr.Error(), http.StatusConflict)
					return
				}
				chef_dbag, nerr := data_bag.New(db_data["name"].(string))
				if nerr != nil {
					JsonErrorReport(w, r, nerr.Error(), nerr.Status())
					return
				}
				chef_dbag.Save()
				db_response["uri"] = util.ObjURL(chef_dbag)
				w.WriteHeader(http.StatusCreated)
			default:
				/* The chef-pedant spec wants this response for
				 * some reason. Mix it up, I guess. */
				JsonErrorReport(w, r, "GET, PUT", http.StatusMethodNotAllowed)
				return
		}
	} else {
		db_name := path_array[1]

		/* chef-pedant is unhappy about not reporting the HTTP status
		 * as 404 by fetching the data bag before we see if the method
		 * is allowed, so do a quick check for that here. */
		if (len(path_array) == 2  && r.Method == "PUT") || (len(path_array) == 3 && r.Method == "POST"){
			JsonErrorReport(w, r, "Method not allowed", http.StatusMethodNotAllowed)
			return
		}
		chef_dbag, err := data_bag.Get(db_name)
		if err != nil {
			var err_msg string
			status := err.Status()
			if r.Method == "POST" {
				/* Posts get a special snowflake message */
				err_msg = fmt.Sprintf("No data bag '%s' could be found. Please create this data bag before adding items to it.", db_name)
			} else {
				if len(path_array) == 3 {
					/* This is nuts. */
					if r.Method == "DELETE" {
						err_msg = fmt.Sprintf("Cannot load data bag %s item %s", db_name, path_array[2])
					} else {
						err_msg = fmt.Sprintf("Cannot load data bag item %s for data bag %s", path_array[2], db_name)
					}
				} else {
					err_msg = err.Error()
				}
			}
			JsonErrorReport(w, r, err_msg, status)
			return
		}
		if len(path_array) == 2 {
			/* getting list of data bag items and creating data bag
			 * items. */
			switch r.Method {
				case "GET":
					for k, _ := range chef_dbag.DataBagItems {
						db_response[k] = util.CustomObjURL(chef_dbag, k)
					}
				case "DELETE":
					/* The chef API docs don't say anything
					 * about this existing, but it does,
					 * and without it you can't delete data
					 * bags at all. */
					db_response["chef_type"] = "data_bag"
					db_response["json_class"] = "Chef::DataBag"
					db_response["name"] = chef_dbag.Name
					err := chef_dbag.Delete()
					if err != nil {
						JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
						return
					}
				case "POST":
					raw_data := data_bag.RawDataBagJson(r.Body)
					dbitem, nerr := chef_dbag.NewDBItem(raw_data)
					if nerr != nil {
						JsonErrorReport(w, r, nerr.Error(), nerr.Status())
						return
					}

					/* The data bag return values are all
					 * kinds of weird. Sometimes it sends
					 * just the raw data, sometimes it sends
					 * the whole object, sometimes a special
					 * snowflake version. Ugh. Have to loop
					 * through to avoid updating the pointer
					 * in the cache by just assigning
					 * dbitem.RawData to db_response. Urk.
					 */
					for k, v := range dbitem.RawData {
						db_response[k] = v
					}
					db_response["data_bag"] = dbitem.DataBagName
					db_response["chef_type"] = dbitem.ChefType
					w.WriteHeader(http.StatusCreated)
				default:
					JsonErrorReport(w, r, "GET, DELETE, POST", http.StatusMethodNotAllowed)
					return
			}
		} else {
			/* getting, editing, and deleting existing data bag items. */
			db_item_name := path_array[2]
			if _, ok := chef_dbag.DataBagItems[db_item_name]; !ok {
				var httperr string
				if r.Method != "DELETE" {
					httperr = fmt.Sprintf("Cannot load data bag item %s for data bag %s", db_item_name, chef_dbag.Name)
				} else {
					httperr = fmt.Sprintf("Cannot load data bag %s item %s", chef_dbag.Name, db_item_name)
				}
				JsonErrorReport(w, r, httperr, http.StatusNotFound)
				return
			}
			switch r.Method {
				case "GET":
					db_response = chef_dbag.DataBagItems[db_item_name].RawData
				case "DELETE":
					dbi := chef_dbag.DataBagItems[db_item_name]
					/* Gotta short circuit this */
					enc := json.NewEncoder(w)
					if err := enc.Encode(&dbi); err != nil {
						JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
						return
					}
					err := chef_dbag.DeleteDBItem(db_item_name)
					if err != nil {
						JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
						return
					}
					return
				case "PUT":
					raw_data := data_bag.RawDataBagJson(r.Body)
					if raw_id, ok := raw_data["id"]; ok {
						switch raw_id := raw_id.(type) {
							case string:
								if raw_id != db_item_name {
									JsonErrorReport(w, r, "DataBagItem name mismatch.", http.StatusBadRequest)
									return
								}
							default:
								JsonErrorReport(w, r, "Bad request", http.StatusBadRequest)
								return
						}
					}
					dbitem, err := chef_dbag.UpdateDBItem(db_item_name, raw_data)
					if err != nil {
						JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
						return
					}
					/* Another weird data bag item response
					 * which isn't at all unusual. */
					for k, v := range dbitem.RawData {
						db_response[k] = v
					}
					db_response["data_bag"] = dbitem.DataBagName
					db_response["chef_type"] = dbitem.ChefType
					db_response["id"] = db_item_name
				default:
					JsonErrorReport(w, r, "GET, DELETE, PUT", http.StatusMethodNotAllowed)
					return
			}
		}
	}

	enc := json.NewEncoder(w)
	if err := enc.Encode(&db_response); err != nil {
		JsonErrorReport(w, r, err.Error(), http.StatusInternalServerError)
	}
}

<|endoftext|>


<commit_before>package server

import (
	"expvar"
	"fmt"
	"net/http"
	"net/http/pprof"

	"github.com/gin-gonic/gin"
)

// Replicated from expvar.go as not public.
func expVars(w http.ResponseWriter, r *http.Request) {
	first := true
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	fmt.Fprintf(w, "{\n")
	expvar.Do(func(kv expvar.KeyValue) {
		if !first {
			fmt.Fprintf(w, ",\n")
		}
		first = false
		fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
	})
	fmt.Fprintf(w, "\n}\n")
}

func profilerSetup(router *gin.Engine, path string) {
	engine := router.Group(path)
	engine.Any("/vars", gin.WrapF(expVars))
	engine.Any("/pprof/", gin.WrapF(pprof.Index))
	engine.Any("/pprof/cmdline", gin.WrapF(pprof.Cmdline))
	engine.Any("/pprof/profile", gin.WrapF(pprof.Profile))
	engine.Any("/pprof/symbol", gin.WrapF(pprof.Symbol))
	engine.Any("/pprof/block", gin.WrapF(pprof.Handler("block").ServeHTTP))
	engine.Any("/pprof/heap", gin.WrapF(pprof.Handler("heap").ServeHTTP))
	engine.Any("/pprof/goroutine", gin.WrapF(pprof.Handler("goroutine").ServeHTTP))
	engine.Any("/pprof/threadcreate", gin.WrapF(pprof.Handler("threadcreate").ServeHTTP))
}
<commit_msg>Remove replicated expvar handler (#805)<commit_after>package server

import (
	"expvar"
	"net/http/pprof"

	"github.com/gin-gonic/gin"
)

func profilerSetup(router *gin.Engine, path string) {
	engine := router.Group(path)
	engine.Any("/vars", gin.WrapF(expvar.Handler().ServeHTTP))
	engine.Any("/pprof/", gin.WrapF(pprof.Index))
	engine.Any("/pprof/cmdline", gin.WrapF(pprof.Cmdline))
	engine.Any("/pprof/profile", gin.WrapF(pprof.Profile))
	engine.Any("/pprof/symbol", gin.WrapF(pprof.Symbol))
	engine.Any("/pprof/block", gin.WrapF(pprof.Handler("block").ServeHTTP))
	engine.Any("/pprof/heap", gin.WrapF(pprof.Handler("heap").ServeHTTP))
	engine.Any("/pprof/goroutine", gin.WrapF(pprof.Handler("goroutine").ServeHTTP))
	engine.Any("/pprof/threadcreate", gin.WrapF(pprof.Handler("threadcreate").ServeHTTP))
}
<|endoftext|>


<commit_before>// JSON Data Storage
package golb

import (
	"encoding/json"
	"io/ioutil"
	"time"
)

type Comment struct {
	Date    time.Time
	Name    string
	Email   string
	URL     string
	Comment string
	Enabled bool
}

type Article struct {
	Date     time.Time
	Title    string
	Slug     string
	Body     string
	Tags     []string
	Enabled  bool
	Author   string
	Comments []Comment
}

type Data struct {
	Articles []Article
	Name     string
}

func Open(name string) *Data {
	d := new(Data)
	d.Name = name
	return d
}

func (d *Data) Read() error {
	data, err := ioutil.ReadFile(d.Name)
	if err != nil {
		return err
	}
	return json.Unmarshal(data, &d.Articles)
}

func (d *Data) Write() error {
	data, err := json.MarshalIndent(d.Articles, "", "	")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(d.Name, data, 0644)
}
<commit_msg>add findarticle<commit_after>// JSON Data Storage
package golb

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"time"
)

type Comment struct {
	Date    time.Time
	Name    string
	Email   string
	URL     string
	Comment string
	Enabled bool
}

type Article struct {
	Date     time.Time
	Title    string
	Slug     string
	Body     string
	Tags     []string
	Enabled  bool
	Author   string
	Comments []Comment
}

type Data struct {
	Articles []Article
	Name     string
}

func Open(name string) *Data {
	d := new(Data)
	d.Name = name
	return d
}

func (d *Data) Read() error {
	data, err := ioutil.ReadFile(d.Name)
	if err != nil {
		return err
	}
	return json.Unmarshal(data, &d.Articles)
}

func (d *Data) Write() error {
	data, err := json.MarshalIndent(d.Articles, "", "	")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(d.Name, data, 0644)
}

func (d *Data) FindArticle(slug string) (Article, error) {
	for _, a := range d.Articles {
		if a.Slug == slug {
			return a, nil
		}
	}
	return Article{}, errors.New("not found")
}
<|endoftext|>