<commit_before>// bstock
// Author: Branislav Blaskovic

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/olekukonko/tablewriter"
	"gopkg.in/yaml.v2"
)

var data = `
description: bu
stocks:
  STOCK:
    ticker: BAASTOCK
    url: url1
    notes: Lol
  UPL:
    ticker: BAASTOCK
    notes: Lol
    url: url2
`

type T struct {
	Stocks map[string]Stock
}

type Stock struct {
	Url      string
	Notes    string
	Currency string
	BuyPrice string
}

func FloatToString(number float64) string {
	return strconv.FormatFloat(number, 'f', 2, 64)
}

func StringToFloat(text string) float64 {
	text = strings.Replace(text, ",", ".", -1)
	reg, _ := regexp.Compile("[^0-9.]+")
	text = reg.ReplaceAllString(text, "")

	num, err := strconv.ParseFloat(text, 32)
	if err != nil {
		fmt.Printf("Error: %s\n", err)
	}

	return num
}

func JoinStrings(str1, str2 string) string {
	return strings.Join([]string{str1, str2}, " ")
}

// GetPrice downloads URL and returns it
func GetPrice(url string) float64 {
	doc, err := goquery.NewDocument(url)
	if err != nil {
		log.Fatal(err)
		return 0.0
	}

	text := doc.Find("#ctl00_BCPP_Celkem_dvCelkem td.num").First().Text()

	return StringToFloat(text)

}

func main() {
	// Config
	path, _ := filepath.Abs("./stocks.yml")
	yamlFile, errFile := ioutil.ReadFile(path)
	if errFile != nil {
		log.Fatalf("Error: %v", errFile)
	}

	t := T{}
	err := yaml.Unmarshal(yamlFile, &t)
	if err != nil {
		log.Fatalf("error: %v", err)
	}

	// This is just for debug
	// fmt.Printf("%#v\n", t)

	// Table to print
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Ticker", "Price", "Buy price"})

	// Stocks cycler
	for ticker, data := range t.Stocks {
		price := GetPrice(data.Url)
		table.Append([]string{ticker, JoinStrings(FloatToString(price), data.Currency), JoinStrings(data.BuyPrice, data.Currency)})

	}

	table.SetAlignment(tablewriter.ALIGN_RIGHT)
	table.Render()
}
<commit_msg>Google finance.<commit_after>// bstock
// Author: Branislav Blaskovic

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/olekukonko/tablewriter"
	"gopkg.in/yaml.v2"
)

type T struct {
	Stocks map[string]Stock
}

type Stock struct {
	Url      string
	Notes    string
	Currency string
	BuyPrice string
}

func FloatToString(number float64) string {
	return strconv.FormatFloat(number, 'f', 2, 64)
}

func StringToFloat(text string) float64 {
	text = strings.Replace(text, ",", ".", -1)
	reg, _ := regexp.Compile("[^0-9.]+")
	text = reg.ReplaceAllString(text, "")

	num, err := strconv.ParseFloat(text, 32)
	if err != nil {
		fmt.Printf("Error: %s\n", err)
	}

	return num
}

func JoinStrings(str1, str2 string) string {
	return strings.Join([]string{str1, str2}, " ")
}

// GetPrice downloads URL and returns it
func GetPrice(url string) float64 {
	doc, err := goquery.NewDocument(url)
	if err != nil {
		log.Fatal(err)
		return 0.0
	}

	var text string

	switch {
	case strings.Contains(url, "bcpp.cz") == true:
		text = doc.Find("#ctl00_BCPP_Celkem_dvCelkem td.num").First().Text()
	case strings.Contains(url, "google.com/finance"):
		text = doc.Find("span.pr span").First().Text()
	}

	return StringToFloat(text)

}

func main() {
	// Config
	path, _ := filepath.Abs("./stocks.yml")
	yamlFile, errFile := ioutil.ReadFile(path)
	if errFile != nil {
		log.Fatalf("Error: %v", errFile)
	}

	t := T{}
	err := yaml.Unmarshal(yamlFile, &t)
	if err != nil {
		log.Fatalf("error: %v", err)
	}

	// This is just for debug
	// fmt.Printf("%#v\n", t)

	// Table to print
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Ticker", "Price", "Buy price", "Notes"})

	// Stocks cycler
	for ticker, data := range t.Stocks {
		price := GetPrice(data.Url)
		table.Append([]string{ticker, JoinStrings(FloatToString(price), data.Currency), JoinStrings(data.BuyPrice, data.Currency), data.Notes})

	}

	table.SetAlignment(tablewriter.ALIGN_RIGHT)
	table.Render()
}
<|endoftext|>


<commit_before>package cairo

//#cgo pkg-config: cairo
//#include <cairo/cairo.h>
import "C"
import "runtime"

type Device interface {
	Resource

	Type() deviceType
	Error() error
}

type cairoDevice struct {
	d *C.cairo_device_t
}

func newCairoDevice(d *C.cairo_device_t) Device {
	if d == nil {
		return nil
	}
	o := &cairoDevice{d: d}
	runtime.SetFinalizer(o, (*cairoDevice).Close)
	return o
}

func (c *cairoDevice) Lock() error {
	return toerr(C.cairo_device_acquire(c.d))
}

func (c *cairoDevice) Unlock() {
	C.cairo_device_release(c.d)
}

func (c *cairoDevice) Close() error {
	if c == nil || c.d == nil {
		return nil
	}
	C.cairo_device_destroy(c.d)
	c.d = nil
	runtime.SetFinalizer(c, nil)
	return nil
}

func (c *cairoDevice) Error() error {
	return toerr(C.cairo_device_status(c.d))
}

func (c *cairoDevice) Type() deviceType {
	return deviceType(C.cairo_device_get_type(c.d))
}
<commit_msg>purely cosmetic tweak, but I feel prettier now<commit_after>package cairo

//#cgo pkg-config: cairo
//#include <cairo/cairo.h>
import "C"

import (
	"runtime"
)

type Device interface {
	Resource

	Type() deviceType
	Error() error
}

type cairoDevice struct {
	d *C.cairo_device_t
}

func newCairoDevice(d *C.cairo_device_t) Device {
	if d == nil {
		return nil
	}
	o := &cairoDevice{d: d}
	runtime.SetFinalizer(o, (*cairoDevice).Close)
	return o
}

func (c *cairoDevice) Lock() error {
	return toerr(C.cairo_device_acquire(c.d))
}

func (c *cairoDevice) Unlock() {
	C.cairo_device_release(c.d)
}

func (c *cairoDevice) Close() error {
	if c == nil || c.d == nil {
		return nil
	}
	C.cairo_device_destroy(c.d)
	c.d = nil
	runtime.SetFinalizer(c, nil)
	return nil
}

func (c *cairoDevice) Error() error {
	return toerr(C.cairo_device_status(c.d))
}

func (c *cairoDevice) Type() deviceType {
	return deviceType(C.cairo_device_get_type(c.d))
}
<|endoftext|>


<commit_before>package openrtb

// 3.2.11 Object: Device
//
// This object provides information pertaining to the device through which the user is interacting. Device
// information includes its hardware, platform, location, and carrier data. The device can refer to a mobile
// handset, a desktop computer, set top box, or other digital device.
type Device struct {

	// Attribute:
	//   ua
	// Type:
	//   string; recommended
	// Description:
	//   Browser user agent string.
	UA string `json:"ua,omitempty"`

	// Attribute:
	//   geo
	// Type:
	//   object; recommended
	// Description:
	//   Location of the device assumed to be the user’s current
	//   location defined by a Geo object (Section 3.2.12).
	Geo *Geo `json:"geo,omitempty"`

	// Attribute:
	//   dnt
	// Type:
	//   integer; recommended
	// Description:
	//   Standard “Do Not Track” flag as set in the header by the
	//   browser, where 0 = tracking is unrestricted, 1 = do not track.
	DNT uint8 `json:"dnt,omitempty"`

	// Attribute:
	//   lmt
	// Type:
	//   integer; recommended
	// Description:
	//   “Limit Ad Tracking” signal commercially endorsed (e.g., iOS,
	//   Android), where 0 = tracking is unrestricted, 1 = tracking must
	//   be limited per commercial guidelines.
	Lmt uint8 `json:"lmt,omitempty"`

	// Attribute:
	//   ip
	// Type:
	//   string; recommended
	// Description:
	//   IPv4 address closest to device.
	IP string `json:"ip,omitempty"`

	// Attribute:
	//   ipv6
	// Type:
	//   string
	// Description:
	//   IP address closest to device as IPv6.
	IPv6 string `json:"ipv6,omitempty"`

	// Attribute:
	//   devicetype
	// Type:
	//   integer
	// Description:
	//   The general type of device. Refer to List 5.17.
	DeviceType uint8 `json:"devicetype,omitempty"`

	// Attribute:
	//   make
	// Type:
	//   string
	// Description:
	//   Device make (e.g., “Apple”).
	Make string `json:"make,omitempty"`

	// Attribute:
	//   model
	// Type:
	//   string
	// Description:
	//   Device model (e.g., “iPhone”).
	Model string `json:"model,omitempty"`

	// Attribute:
	//   os
	// Type:
	//   string
	// Description:
	//   Device operating system (e.g., “iOS”).
	OS string `json:"os,omitempty"`

	// Attribute:
	//   osv
	// Type:
	//   string
	// Description:
	//   Device operating system version (e.g., “3.1.2”).
	OSV string `json:"osv,omitempty"`

	// Attribute:
	//   hwv
	// Type:
	//   string
	// Description:
	//   Hardware version of the device (e.g., “5S” for iPhone 5S).
	HWV string `json:"hwv,omitempty"`

	// Attribute:
	//   h
	// Type:
	//   integer
	// Description:
	//   Physical height of the screen in pixels.
	H uint64 `json:"h,omitempty"`

	// Attribute:
	//   w
	// Type:
	//   integer
	// Description:
	//   Physical width of the screen in pixels.
	W uint64 `json:"w,omitempty"`

	// Attribute:
	//   ppi
	// Type:
	//   integer
	// Description:
	//   Screen size as pixels per linear inch.
	PPI uint64 `json:"ppi,omitempty"`

	// Attribute:
	//   pxratio
	// Type:
	//   float
	// Description:
	//   The ratio of physical pixels to device independent pixels.
	PxRatio float64 `json:"pxratio,omitempty"`

	// Attribute:
	//   js
	// Type:
	//   integer
	// Description:
	//   Support for JavaScript, where 0 = no, 1 = yes.
	JS uint8 `json:"js,omitempty"`

	// Attribute:
	//   flashver
	// Type:
	//   string
	// Description:
	//   Version of Flash supported by the browser.
	FlashVer string `json:"flashver,omitempty"`

	// Attribute:
	//   language
	// Type:
	//   string
	// Description:
	//   Browser language using ISO-639-1-alpha-2.
	Language string `json:"language,omitempty"`

	// Attribute:
	//   carrier
	// Type:
	//   string
	// Description:
	//   Carrier or ISP (e.g., “VERIZON”). “WIFI” is often used in mobile
	//   to indicate high bandwidth (e.g., video friendly vs. cellular).
	//   connectiontype integer Network connection type. Refer to List 5.18.
	Carrier string `json:"carrier,omitempty"`

	// Attribute:
	//   ifa
	// Type:
	//   string
	// Description:
	//   ID sanctioned for advertiser use in the clear (i.e., not hashed).
	IFA string `json:"ifa,omitempty"`

	// Attribute:
	//   didsha1
	// Type:
	//   string
	// Description:
	//   Hardware device ID (e.g., IMEI); hashed via SHA1.
	DIDSHA1 string `json:"didsha1,omitempty"`

	// Attribute:
	//   didmd5
	// Type:
	//   string
	// Description:
	//  Hardware device ID (e.g., IMEI); hashed via MD5.
	DIDMD5 string `json:"didmd5,omitempty"`

	// Attribute:
	//   dpidsha1
	// Type:
	//   string
	// Description:
	//   Platform device ID (e.g., Android ID); hashed via SHA1.
	DPIDSHA1 string `json:"dpidsha1,omitempty"`

	// Attribute:
	//   dpidmd5
	// Type:
	//   string
	// Description:
	//   Platform device ID (e.g., Android ID); hashed via MD5.
	DPIDMD5 string `json:"dpidmd5,omitempty"`

	// Attribute:
	//   macsha1
	// Type:
	//   string
	// Description:
	//   MAC address of the device; hashed via SHA1.
	MACSHA1 string `json:"macsha1,omitempty"`

	// Attribute:
	//   macmd5
	// Type:
	//   string
	// Description:
	//   MAC address of the device; hashed via MD5.
	MACMD5 string `json:"macmd5,omitempty"`

	// Attribute:
	//   ext
	// Type:
	//   object
	// Description:
	//   Placeholder for exchange-specific extensions to OpenRTB.
	Ext Ext `json:"ext,omitempty"`
}
<commit_msg>Device: added 'connectiontype'<commit_after>package openrtb

// 3.2.11 Object: Device
//
// This object provides information pertaining to the device through which the user is interacting. Device
// information includes its hardware, platform, location, and carrier data. The device can refer to a mobile
// handset, a desktop computer, set top box, or other digital device.
type Device struct {

	// Attribute:
	//   ua
	// Type:
	//   string; recommended
	// Description:
	//   Browser user agent string.
	UA string `json:"ua,omitempty"`

	// Attribute:
	//   geo
	// Type:
	//   object; recommended
	// Description:
	//   Location of the device assumed to be the user’s current
	//   location defined by a Geo object (Section 3.2.12).
	Geo *Geo `json:"geo,omitempty"`

	// Attribute:
	//   dnt
	// Type:
	//   integer; recommended
	// Description:
	//   Standard “Do Not Track” flag as set in the header by the
	//   browser, where 0 = tracking is unrestricted, 1 = do not track.
	DNT uint8 `json:"dnt,omitempty"`

	// Attribute:
	//   lmt
	// Type:
	//   integer; recommended
	// Description:
	//   “Limit Ad Tracking” signal commercially endorsed (e.g., iOS,
	//   Android), where 0 = tracking is unrestricted, 1 = tracking must
	//   be limited per commercial guidelines.
	Lmt uint8 `json:"lmt,omitempty"`

	// Attribute:
	//   ip
	// Type:
	//   string; recommended
	// Description:
	//   IPv4 address closest to device.
	IP string `json:"ip,omitempty"`

	// Attribute:
	//   ipv6
	// Type:
	//   string
	// Description:
	//   IP address closest to device as IPv6.
	IPv6 string `json:"ipv6,omitempty"`

	// Attribute:
	//   devicetype
	// Type:
	//   integer
	// Description:
	//   The general type of device. Refer to List 5.17.
	DeviceType uint8 `json:"devicetype,omitempty"`

	// Attribute:
	//   make
	// Type:
	//   string
	// Description:
	//   Device make (e.g., “Apple”).
	Make string `json:"make,omitempty"`

	// Attribute:
	//   model
	// Type:
	//   string
	// Description:
	//   Device model (e.g., “iPhone”).
	Model string `json:"model,omitempty"`

	// Attribute:
	//   os
	// Type:
	//   string
	// Description:
	//   Device operating system (e.g., “iOS”).
	OS string `json:"os,omitempty"`

	// Attribute:
	//   osv
	// Type:
	//   string
	// Description:
	//   Device operating system version (e.g., “3.1.2”).
	OSV string `json:"osv,omitempty"`

	// Attribute:
	//   hwv
	// Type:
	//   string
	// Description:
	//   Hardware version of the device (e.g., “5S” for iPhone 5S).
	HWV string `json:"hwv,omitempty"`

	// Attribute:
	//   h
	// Type:
	//   integer
	// Description:
	//   Physical height of the screen in pixels.
	H uint64 `json:"h,omitempty"`

	// Attribute:
	//   w
	// Type:
	//   integer
	// Description:
	//   Physical width of the screen in pixels.
	W uint64 `json:"w,omitempty"`

	// Attribute:
	//   ppi
	// Type:
	//   integer
	// Description:
	//   Screen size as pixels per linear inch.
	PPI uint64 `json:"ppi,omitempty"`

	// Attribute:
	//   pxratio
	// Type:
	//   float
	// Description:
	//   The ratio of physical pixels to device independent pixels.
	PxRatio float64 `json:"pxratio,omitempty"`

	// Attribute:
	//   js
	// Type:
	//   integer
	// Description:
	//   Support for JavaScript, where 0 = no, 1 = yes.
	JS uint8 `json:"js,omitempty"`

	// Attribute:
	//   flashver
	// Type:
	//   string
	// Description:
	//   Version of Flash supported by the browser.
	FlashVer string `json:"flashver,omitempty"`

	// Attribute:
	//   language
	// Type:
	//   string
	// Description:
	//   Browser language using ISO-639-1-alpha-2.
	Language string `json:"language,omitempty"`

	// Attribute:
	//   carrier
	// Type:
	//   string
	// Description:
	//   Carrier or ISP (e.g., “VERIZON”). “WIFI” is often used in mobile
	//   to indicate high bandwidth (e.g., video friendly vs. cellular).
	//   connectiontype integer Network connection type. Refer to List 5.18.
	Carrier string `json:"carrier,omitempty"`

	// Attribute:
	//   connectiontype
	// Type:
	//   integer
	// Description:
	//   Network connection type. Refer to List 5.18.
	Connectiontype uint8 `json:"connectiontype,omitempty"`

	// Attribute:
	//   ifa
	// Type:
	//   string
	// Description:
	//   ID sanctioned for advertiser use in the clear (i.e., not hashed).
	IFA string `json:"ifa,omitempty"`

	// Attribute:
	//   didsha1
	// Type:
	//   string
	// Description:
	//   Hardware device ID (e.g., IMEI); hashed via SHA1.
	DIDSHA1 string `json:"didsha1,omitempty"`

	// Attribute:
	//   didmd5
	// Type:
	//   string
	// Description:
	//  Hardware device ID (e.g., IMEI); hashed via MD5.
	DIDMD5 string `json:"didmd5,omitempty"`

	// Attribute:
	//   dpidsha1
	// Type:
	//   string
	// Description:
	//   Platform device ID (e.g., Android ID); hashed via SHA1.
	DPIDSHA1 string `json:"dpidsha1,omitempty"`

	// Attribute:
	//   dpidmd5
	// Type:
	//   string
	// Description:
	//   Platform device ID (e.g., Android ID); hashed via MD5.
	DPIDMD5 string `json:"dpidmd5,omitempty"`

	// Attribute:
	//   macsha1
	// Type:
	//   string
	// Description:
	//   MAC address of the device; hashed via SHA1.
	MACSHA1 string `json:"macsha1,omitempty"`

	// Attribute:
	//   macmd5
	// Type:
	//   string
	// Description:
	//   MAC address of the device; hashed via MD5.
	MACMD5 string `json:"macmd5,omitempty"`

	// Attribute:
	//   ext
	// Type:
	//   object
	// Description:
	//   Placeholder for exchange-specific extensions to OpenRTB.
	Ext Ext `json:"ext,omitempty"`
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"

	"github.com/fsouza/go-dockerclient"
)

func command(config *docker.HealthConfig) []string {
	switch config.Test[0] {
	case "CMD":
		return config.Test[1:len(config.Test)]
	case "CMD-SHELL":
		return append([]string{"/bin/sh", "-c"}, config.Test[1:len(config.Test)]...)
	default:
		return []string{"echo", "Healthcheck", config.Test[0]}
	}
}

func healtcheck(client *docker.Client, container *docker.Container) error {
	exec, err := client.CreateExec(docker.CreateExecOptions{
		AttachStdin:  false,
		AttachStdout: true,
		AttachStderr: true,
		Tty:          false,
		Cmd:          command(container.Config.Healthcheck),
		Container:    container.ID,
	})
	if err != nil {
		return err
	}

	var stdout, stderr bytes.Buffer
	err = client.StartExec(exec.ID, docker.StartExecOptions{
		OutputStream: &stdout,
		ErrorStream:  &stderr,
		RawTerminal:  true,
	})
	if err != nil {
		return err
	}

	fmt.Printf("stdout: '%s'\n", stdout.String())
	return nil
}

func main() {
	endpoint := flag.String("endpoint", "unix:///var/run/docker.sock", "Docker socket")

	client, err := docker.NewClient(*endpoint)
	if err != nil {
		fmt.Printf("Error (%s) %s\n", *endpoint, err.Error())
		os.Exit(1)
	}

	events := make(chan *docker.APIEvents)
	client.AddEventListener(events)
	fmt.Println("Listening...")

	for {
		select {
		case event := <-events:
			switch event.Status {
			case "start":
				fmt.Printf("> START %s: %s\n", event.From, event.ID)

				container, err := client.InspectContainer(event.ID)
				if err != nil {
					fmt.Printf("  %s\n", err.Error())
					break
				}

				fmt.Printf("  %+v\n", container.Config.Healthcheck)
				if container.Config.Healthcheck == nil || len(container.Config.Healthcheck.Test) == 0 {
					fmt.Printf("  No Healthcheck, assuming container started.\n")
					break
				}

				healtcheck(client, container)

			case "stop":
				fmt.Printf("> STOP %s: %s\n", event.From, event.ID)
			case "die":
				fmt.Printf("> DIE %s: %s\n", event.From, event.ID)
			}
		}
	}
}
<commit_msg>Check for exit code<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/fsouza/go-dockerclient"
)

const TRIES = 10

type Healthcheck struct {
	Result int
	Output string
}

func command(config *docker.HealthConfig) []string {
	switch config.Test[0] {
	case "CMD":
		return config.Test[1:len(config.Test)]
	case "CMD-SHELL":
		return append([]string{"/bin/sh", "-c"}, config.Test[1:len(config.Test)]...)
	default:
		return []string{"echo", "Healthcheck", config.Test[0]}
	}
}

func healthcheck(client *docker.Client, container *docker.Container) (*Healthcheck, error) {
	exec, err := client.CreateExec(docker.CreateExecOptions{
		AttachStdin:  false,
		AttachStdout: true,
		AttachStderr: true,
		Tty:          false,
		Cmd:          command(container.Config.Healthcheck),
		Container:    container.ID,
	})
	if err != nil {
		return nil, err
	}

	var stdout, stderr bytes.Buffer
	err = client.StartExec(exec.ID, docker.StartExecOptions{
		OutputStream: &stdout,
		ErrorStream:  &stderr,
		RawTerminal:  true,
	})
	if err != nil {
		return nil, err
	}

	inspect, err := client.InspectExec(exec.ID)
	if err != nil {
		return nil, err
	}

	return &Healthcheck{Result: inspect.ExitCode, Output: stdout.String()}, nil
}

func wait(client *docker.Client, ID string) error {
	container, err := client.InspectContainer(ID)
	if err != nil {
		return err
	}

	fmt.Printf("  %+v\n", container.Config.Healthcheck)
	if container.Config.Healthcheck == nil || len(container.Config.Healthcheck.Test) == 0 {
		fmt.Printf("  No Healthcheck, assuming container started.\n")
		return nil
	}

	tries := TRIES
	for tries > 0 {
		check, err := healthcheck(client, container)
		fmt.Printf("  %+v.\n", check)
		if err != nil {
			return err
		}

		if check.Result == 0 {
			return nil
		}

		time.Sleep(1 * time.Second)
		tries--
	}
	return fmt.Errorf("Timed out")
}

func main() {
	endpoint := flag.String("endpoint", "unix:///var/run/docker.sock", "Docker socket")

	client, err := docker.NewClient(*endpoint)
	if err != nil {
		fmt.Printf("Error (%s) %s\n", *endpoint, err.Error())
		os.Exit(1)
	}

	events := make(chan *docker.APIEvents)
	client.AddEventListener(events)
	fmt.Println("Listening...")

	for {
		select {
		case event := <-events:
			switch event.Status {
			case "start":
				fmt.Printf("> START %s: %s\n", event.From, event.ID)

				err := wait(client, event.ID)
				if err != nil {
					fmt.Printf("  %s\n", err.Error())
				} else {
					fmt.Printf("  Up and running!\n")
				}

			case "stop":
				fmt.Printf("> STOP %s: %s\n", event.From, event.ID)
			case "die":
				fmt.Printf("> DIE %s: %s\n", event.From, event.ID)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/Luzifer/dockermanager/config"
	"github.com/Luzifer/go_helpers/str"
	log "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
)

const (
	labelIsManaged   = "io.luzifer.dockermanager.managed"
	labelConfigHash  = "io.luzifer.dockermanager.cfghash"
	labelIsScheduled = "io.luzifer.dockermanager.scheduler"

	strTrue = "true"
)

var (
	lastStartContainerCall = time.Now()

	pullLock     = map[string]bool{}
	pullLockLock sync.RWMutex
)

func getImages(dangling bool) []docker.APIImages {
	images, err := dockerClient.ListImages(docker.ListImagesOptions{
		All: false,
	})

	if err != nil {
		log.Errorf("Unable to list images: %s", err)
		return nil
	}

	nowSeconds := time.Now().Unix()
	response := []docker.APIImages{}

	for _, v := range images {
		if dangling {
			if len(v.RepoTags) == 1 && v.RepoTags[0] == "<none>:<none>" && (nowSeconds-v.Created) > 3600 {
				response = append(response, v)
			}
		} else {
			if len(v.RepoTags) != 1 || v.RepoTags[0] != "<none>:<none>" {
				response = append(response, v)
			}
		}
	}

	return response
}

func cleanDangling() {
	images := getImages(true)
	for _, v := range images {
		log.Debugf("Removing dangling image: %s", v.ID)
		dockerClient.RemoveImage(v.ID)
	}
}

func removeNotRequiredImages() {
	if !params.ManageFullHost {
		// If we're not responsible for the full host don't remove images
		return
	}

	required := []string{}
	for _, v := range *cfg {
		if str.StringInSlice(hostname, v.Hosts) || str.StringInSlice("ALL", v.Hosts) {
			required = append(required, fmt.Sprintf("%s:%s", v.Image, v.Tag))
		}
	}

	currentImages := getImages(false)
	for _, i := range currentImages {
		found := false
		for _, t := range i.RepoTags {
			for _, r := range required {
				if r == t {
					found = true
				}
			}
		}

		if !found {
			log.Debugf("Removing not required image: %s", i.ID)
			dockerClient.RemoveImage(i.ID)
		}
	}
}

func refreshImages() {
	for _, v := range *cfg {
		if str.StringInSlice(hostname, v.Hosts) || str.StringInSlice("ALL", v.Hosts) {
			pullImage(v.Image, v.Tag)
		}
	}
}

func pullImage(image, tag string) {
	pullLockLock.Lock()
	if pullLock[image+":"+tag] {
		log.Debugf("Image %q is already pulling, starting no new pull", image+":"+tag)
		pullLockLock.Unlock()
		return
	}
	pullLock[image+":"+tag] = true
	pullLockLock.Unlock()

	defer func() {
		pullLockLock.Lock()
		pullLock[image+":"+tag] = false
		pullLockLock.Unlock()
	}()

	auth := docker.AuthConfiguration{}

	reginfo := strings.SplitN(image, "/", 2)
	if len(reginfo) == 2 {
		for s, a := range authConfig.Configs {
			if strings.Contains(s, reginfo[0]) {
				auth = a
			}
		}
	}

	log.Debugf("Refreshing repo %s:%s...", image, tag)
	if err := dockerClient.PullImage(docker.PullImageOptions{
		Repository: image,
		Tag:        tag,
	}, auth); err != nil {
		log.WithFields(log.Fields{
			"repo": image + ":" + tag,
		}).Errorf("An error occurred while image pulling: %s", err)
	}
}

func bootContainer(name string, cfg config.ContainerConfig) {
	var (
		container *docker.Container
		err       error
	)

	cs, err := cfg.Checksum()
	if err != nil {
		log.Errorf("Unable to calculate checksum for container %q: %s", name, err)
		return
	}

	labels := map[string]string{}
	if labels != nil {
		for k, v := range cfg.Labels {
			labels[k] = v
		}
	}
	labels[labelConfigHash] = cs
	labels[labelIsManaged] = strTrue

	if cfg.StartTimes != "" {
		labels[labelIsScheduled] = strTrue
	}

	volumes, binds := parseMounts(cfg.Volumes)

	newcfg := &docker.Config{
		AttachStdin:  false,
		AttachStdout: true,
		AttachStderr: true,
		Image:        strings.Join([]string{cfg.Image, cfg.Tag}, ":"),
		Env:          cfg.Environment,
		Cmd:          cfg.Command,
		Labels:       labels,
		Volumes:      volumes,
	}

	hostConfig := &docker.HostConfig{
		Binds:        binds,
		Links:        cfg.Links,
		Privileged:   false,
		PortBindings: make(map[docker.Port][]docker.PortBinding),
		CapAdd:       cfg.AddCapabilities,
	}

	for _, v := range cfg.Ports {
		s := strings.Split(v.Local, ":")
		hostConfig.PortBindings[docker.Port(v.Container)] = []docker.PortBinding{{
			HostIP:   s[0],
			HostPort: s[1],
		}}
	}

	log.Debugf("Creating container %s", name)
	container, err = dockerClient.CreateContainer(docker.CreateContainerOptions{
		Name:       name,
		Config:     newcfg,
		HostConfig: hostConfig,
	})

	if err != nil {
		switch {
		case strings.Contains(err.Error(), "is already in use by container"):
			go cleanContainers()
		case strings.Contains(err.Error(), "already exists"):
			go cleanContainers()
		}
		log.Errorf("Unable to create container '%s': %s", name, err)
		return
	}

	log.Debugf("Starting container %s", container.Name)
	dockerClient.StartContainer(container.Name, nil)
}

func listContainers(listDeadContainers bool) ([]docker.APIContainers, error) {
	return dockerClient.ListContainers(docker.ListContainersOptions{
		All: listDeadContainers,
	})
}

func getExpectedRunningNames() []string {
	expectedRunning := []string{}
	for name, containerCfg := range *cfg {
		if containerCfg.ShouldBeRunning(hostname, lastStartContainerCall) {
			expectedRunning = append(expectedRunning, name)
		}
	}
	return expectedRunning
}

func stopUnexpectedContainers() {
	expectedRunning := getExpectedRunningNames()

	// Stop containers not expected to be running
	currentRunning, err := listContainers(false)
	if err != nil {
		log.Errorf("Unable to list running containers: %s", err)
		return
	}
	for _, v := range currentRunning {
		allowed := false

		containerDetails, err := dockerClient.InspectContainer(v.ID)
		if err != nil {
			log.Errorf("Unable to inspect container %q: %s", v.ID, err)
			continue
		}

		_, isManaged := containerDetails.Config.Labels[labelIsManaged]
		if !params.ManageFullHost && !isManaged {
			allowed = true
		}

		if containerDetails.Config.Labels[labelIsScheduled] == strTrue {
			// If it's a scheduled container keep it running
			allowed = true
		}

		for _, n := range v.Names {
			if str.StringInSlice(strings.Trim(n, "/"), expectedRunning) {
				allowed = true
			}
		}
		if !allowed {
			log.Debugf("Stopping container %s as it is not expected to be running.", v.ID)
			if err := dockerClient.StopContainer(v.ID, 5); err != nil {
				log.Errorf("Unable to stop container %q", v.ID)
				continue
			}
		}
	}
}

func removeDeprecatedContainers() {
	expectedRunning := getExpectedRunningNames()

	// Stop containers who are of deprecated images
	currentRunning, err := listContainers(false)
	if err != nil {
		log.Errorf("Unable to list running containers: %s", err)
		return
	}
	images := getImages(false)
	for _, n := range expectedRunning {
		repoName := strings.Join([]string{(*cfg)[n].Image, (*cfg)[n].Tag}, ":")
		currentImageID := "0"
		for _, i := range images {
			if str.StringInSlice(repoName, i.RepoTags) {
				currentImageID = i.ID
			}
		}
		if currentImageID == "0" {
			log.Debugf("Found no image ID for repo %q, starting background pull", strings.Join([]string{(*cfg)[n].Image, (*cfg)[n].Tag}, ":"))
			go pullImage((*cfg)[n].Image, (*cfg)[n].Tag)
			continue
		}
		for _, v := range currentRunning {
			containerDetails, err := dockerClient.InspectContainer(v.ID)
			if err != nil {
				log.Errorf("Unable to inspect container %q: %s", v.ID, err)
				continue
			}

			cs, err := (*cfg)[n].Checksum()
			if err != nil {
				log.Errorf("Unable to calculate checksum for container %q: %s", v.ID, err)
				continue
			}

			if str.StringInSlice(fmt.Sprintf("/%s", n), v.Names) {
				needsUpdate := false
				if !strings.HasPrefix(currentImageID, containerDetails.Image) {
					log.Infof("Container %s has a new image version.", n)
					needsUpdate = true
				}
				if containerDetails.Config.Labels[labelConfigHash] != cs {
					log.Infof("Container %s has a configuration update.", n)
					needsUpdate = true
				}
				if needsUpdate && !(*cfg)[n].UpdateAllowedAt(time.Now()) {
					log.Infof("Image %s has update but container %s (%s) is not allowed to update now.", v.Image, n, v.ID)
					needsUpdate = false
				}

				if needsUpdate {
					stopWaitTime := (*cfg)[n].StopTimeout
					if stopWaitTime == 0 {
						stopWaitTime = 5
					}

					log.Debugf("Image: %s Current: %s", containerDetails.Image, currentImageID)
					log.Debugf("Stopping deprecated container %s", v.ID)
					if err = dockerClient.StopContainer(v.ID, stopWaitTime); err != nil {
						log.Errorf("Unable to stop container %q", v.ID)
						continue
					}

					log.Debugf("Removing deprecated container %s", v.ID)
					dockerClient.RemoveContainer(docker.RemoveContainerOptions{
						ID: v.ID,
					})

					if currentRunning, err = listContainers(false); err != nil {
						log.Errorf("Unable to list running containers: %s", err)
						return
					}
				}
			}
		}
	}
}

func startExpectedContainers() {
	expectedRunning := getExpectedRunningNames()
	lastStartContainerCall = time.Now()

	// Start expected containers
	currentRunning, err := listContainers(false)
	if err != nil {
		log.Errorf("Unable to list running containers: %s", err)
		return
	}
	runningNames := []string{}
	for _, v := range currentRunning {
		for _, n := range v.Names {
			runningNames = append(runningNames, strings.Trim(n, "/"))
		}
	}
	for _, n := range expectedRunning {
		if !str.StringInSlice(n, runningNames) {
			bootContainer(n, (*cfg)[n])
		}
	}
}

func cleanContainers() {
	runningContainers, err := listContainers(true)
	if err != nil {
		log.Errorf("Unable to list running containers: %s", err)
		return
	}

	for _, v := range runningContainers {
		_, isManaged := v.Labels[labelIsManaged]

		if strings.HasPrefix(v.Status, "Exited") || strings.HasPrefix(v.Status, "Dead") || (strings.HasPrefix(v.Status, "Created") && isManaged) {
			log.Debugf("Removing container %s (Status %s)", v.ID, v.Status)
			if err := dockerClient.RemoveContainer(docker.RemoveContainerOptions{
				ID:    v.ID,
				Force: true,
			}); err != nil {
				log.Errorf("Unable to remove container %s (Status %s): %s", v.ID, v.Status, err)
			}
		}
	}
}

func parseMounts(mountIn []string) (volumes map[string]struct{}, binds []string) {
	volumes = make(map[string]struct{})
	for _, m := range mountIn {
		if len(m) == 0 {
			continue
		}

		parts := strings.Split(m, ":")
		if len(parts) != 2 && len(parts) != 3 {
			log.Errorf("Invalid default mount: %s", m)
			continue
		}

		binds = append(binds, m)
		volumes[parts[1]] = struct{}{}
	}

	return
}
<commit_msg>Log starts as info<commit_after>package main

import (
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/Luzifer/dockermanager/config"
	"github.com/Luzifer/go_helpers/str"
	log "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
)

const (
	labelIsManaged   = "io.luzifer.dockermanager.managed"
	labelConfigHash  = "io.luzifer.dockermanager.cfghash"
	labelIsScheduled = "io.luzifer.dockermanager.scheduler"

	strTrue = "true"
)

var (
	lastStartContainerCall = time.Now()

	pullLock     = map[string]bool{}
	pullLockLock sync.RWMutex
)

func getImages(dangling bool) []docker.APIImages {
	images, err := dockerClient.ListImages(docker.ListImagesOptions{
		All: false,
	})

	if err != nil {
		log.Errorf("Unable to list images: %s", err)
		return nil
	}

	nowSeconds := time.Now().Unix()
	response := []docker.APIImages{}

	for _, v := range images {
		if dangling {
			if len(v.RepoTags) == 1 && v.RepoTags[0] == "<none>:<none>" && (nowSeconds-v.Created) > 3600 {
				response = append(response, v)
			}
		} else {
			if len(v.RepoTags) != 1 || v.RepoTags[0] != "<none>:<none>" {
				response = append(response, v)
			}
		}
	}

	return response
}

func cleanDangling() {
	images := getImages(true)
	for _, v := range images {
		log.Debugf("Removing dangling image: %s", v.ID)
		dockerClient.RemoveImage(v.ID)
	}
}

func removeNotRequiredImages() {
	if !params.ManageFullHost {
		// If we're not responsible for the full host don't remove images
		return
	}

	required := []string{}
	for _, v := range *cfg {
		if str.StringInSlice(hostname, v.Hosts) || str.StringInSlice("ALL", v.Hosts) {
			required = append(required, fmt.Sprintf("%s:%s", v.Image, v.Tag))
		}
	}

	currentImages := getImages(false)
	for _, i := range currentImages {
		found := false
		for _, t := range i.RepoTags {
			for _, r := range required {
				if r == t {
					found = true
				}
			}
		}

		if !found {
			log.Debugf("Removing not required image: %s", i.ID)
			dockerClient.RemoveImage(i.ID)
		}
	}
}

func refreshImages() {
	for _, v := range *cfg {
		if str.StringInSlice(hostname, v.Hosts) || str.StringInSlice("ALL", v.Hosts) {
			pullImage(v.Image, v.Tag)
		}
	}
}

func pullImage(image, tag string) {
	pullLockLock.Lock()
	if pullLock[image+":"+tag] {
		log.Debugf("Image %q is already pulling, starting no new pull", image+":"+tag)
		pullLockLock.Unlock()
		return
	}
	pullLock[image+":"+tag] = true
	pullLockLock.Unlock()

	defer func() {
		pullLockLock.Lock()
		pullLock[image+":"+tag] = false
		pullLockLock.Unlock()
	}()

	auth := docker.AuthConfiguration{}

	reginfo := strings.SplitN(image, "/", 2)
	if len(reginfo) == 2 {
		for s, a := range authConfig.Configs {
			if strings.Contains(s, reginfo[0]) {
				auth = a
			}
		}
	}

	log.Debugf("Refreshing repo %s:%s...", image, tag)
	if err := dockerClient.PullImage(docker.PullImageOptions{
		Repository: image,
		Tag:        tag,
	}, auth); err != nil {
		log.WithFields(log.Fields{
			"repo": image + ":" + tag,
		}).Errorf("An error occurred while image pulling: %s", err)
	}
}

func bootContainer(name string, cfg config.ContainerConfig) {
	var (
		container *docker.Container
		err       error
	)

	cs, err := cfg.Checksum()
	if err != nil {
		log.Errorf("Unable to calculate checksum for container %q: %s", name, err)
		return
	}

	labels := map[string]string{}
	if labels != nil {
		for k, v := range cfg.Labels {
			labels[k] = v
		}
	}
	labels[labelConfigHash] = cs
	labels[labelIsManaged] = strTrue

	if cfg.StartTimes != "" {
		labels[labelIsScheduled] = strTrue
	}

	volumes, binds := parseMounts(cfg.Volumes)

	newcfg := &docker.Config{
		AttachStdin:  false,
		AttachStdout: true,
		AttachStderr: true,
		Image:        strings.Join([]string{cfg.Image, cfg.Tag}, ":"),
		Env:          cfg.Environment,
		Cmd:          cfg.Command,
		Labels:       labels,
		Volumes:      volumes,
	}

	hostConfig := &docker.HostConfig{
		Binds:        binds,
		Links:        cfg.Links,
		Privileged:   false,
		PortBindings: make(map[docker.Port][]docker.PortBinding),
		CapAdd:       cfg.AddCapabilities,
	}

	for _, v := range cfg.Ports {
		s := strings.Split(v.Local, ":")
		hostConfig.PortBindings[docker.Port(v.Container)] = []docker.PortBinding{{
			HostIP:   s[0],
			HostPort: s[1],
		}}
	}

	log.Debugf("Creating container %s", name)
	container, err = dockerClient.CreateContainer(docker.CreateContainerOptions{
		Name:       name,
		Config:     newcfg,
		HostConfig: hostConfig,
	})

	if err != nil {
		switch {
		case strings.Contains(err.Error(), "is already in use by container"):
			go cleanContainers()
		case strings.Contains(err.Error(), "already exists"):
			go cleanContainers()
		}
		log.Errorf("Unable to create container '%s': %s", name, err)
		return
	}

	log.Infof("Starting container %s", container.Name)
	dockerClient.StartContainer(container.Name, nil)
}

func listContainers(listDeadContainers bool) ([]docker.APIContainers, error) {
	return dockerClient.ListContainers(docker.ListContainersOptions{
		All: listDeadContainers,
	})
}

func getExpectedRunningNames() []string {
	expectedRunning := []string{}
	for name, containerCfg := range *cfg {
		if containerCfg.ShouldBeRunning(hostname, lastStartContainerCall) {
			expectedRunning = append(expectedRunning, name)
		}
	}
	return expectedRunning
}

func stopUnexpectedContainers() {
	expectedRunning := getExpectedRunningNames()

	// Stop containers not expected to be running
	currentRunning, err := listContainers(false)
	if err != nil {
		log.Errorf("Unable to list running containers: %s", err)
		return
	}
	for _, v := range currentRunning {
		allowed := false

		containerDetails, err := dockerClient.InspectContainer(v.ID)
		if err != nil {
			log.Errorf("Unable to inspect container %q: %s", v.ID, err)
			continue
		}

		_, isManaged := containerDetails.Config.Labels[labelIsManaged]
		if !params.ManageFullHost && !isManaged {
			allowed = true
		}

		if containerDetails.Config.Labels[labelIsScheduled] == strTrue {
			// If it's a scheduled container keep it running
			allowed = true
		}

		for _, n := range v.Names {
			if str.StringInSlice(strings.Trim(n, "/"), expectedRunning) {
				allowed = true
			}
		}
		if !allowed {
			log.Debugf("Stopping container %s as it is not expected to be running.", v.ID)
			if err := dockerClient.StopContainer(v.ID, 5); err != nil {
				log.Errorf("Unable to stop container %q", v.ID)
				continue
			}
		}
	}
}

func removeDeprecatedContainers() {
	expectedRunning := getExpectedRunningNames()

	// Stop containers who are of deprecated images
	currentRunning, err := listContainers(false)
	if err != nil {
		log.Errorf("Unable to list running containers: %s", err)
		return
	}
	images := getImages(false)
	for _, n := range expectedRunning {
		repoName := strings.Join([]string{(*cfg)[n].Image, (*cfg)[n].Tag}, ":")
		currentImageID := "0"
		for _, i := range images {
			if str.StringInSlice(repoName, i.RepoTags) {
				currentImageID = i.ID
			}
		}
		if currentImageID == "0" {
			log.Debugf("Found no image ID for repo %q, starting background pull", strings.Join([]string{(*cfg)[n].Image, (*cfg)[n].Tag}, ":"))
			go pullImage((*cfg)[n].Image, (*cfg)[n].Tag)
			continue
		}
		for _, v := range currentRunning {
			containerDetails, err := dockerClient.InspectContainer(v.ID)
			if err != nil {
				log.Errorf("Unable to inspect container %q: %s", v.ID, err)
				continue
			}

			cs, err := (*cfg)[n].Checksum()
			if err != nil {
				log.Errorf("Unable to calculate checksum for container %q: %s", v.ID, err)
				continue
			}

			if str.StringInSlice(fmt.Sprintf("/%s", n), v.Names) {
				needsUpdate := false
				if !strings.HasPrefix(currentImageID, containerDetails.Image) {
					log.Infof("Container %s has a new image version.", n)
					needsUpdate = true
				}
				if containerDetails.Config.Labels[labelConfigHash] != cs {
					log.Infof("Container %s has a configuration update.", n)
					needsUpdate = true
				}
				if needsUpdate && !(*cfg)[n].UpdateAllowedAt(time.Now()) {
					log.Infof("Image %s has update but container %s (%s) is not allowed to update now.", v.Image, n, v.ID)
					needsUpdate = false
				}

				if needsUpdate {
					stopWaitTime := (*cfg)[n].StopTimeout
					if stopWaitTime == 0 {
						stopWaitTime = 5
					}

					log.Debugf("Image: %s Current: %s", containerDetails.Image, currentImageID)
					log.Debugf("Stopping deprecated container %s", v.ID)
					if err = dockerClient.StopContainer(v.ID, stopWaitTime); err != nil {
						log.Errorf("Unable to stop container %q", v.ID)
						continue
					}

					log.Debugf("Removing deprecated container %s", v.ID)
					dockerClient.RemoveContainer(docker.RemoveContainerOptions{
						ID: v.ID,
					})

					if currentRunning, err = listContainers(false); err != nil {
						log.Errorf("Unable to list running containers: %s", err)
						return
					}
				}
			}
		}
	}
}

func startExpectedContainers() {
	expectedRunning := getExpectedRunningNames()
	lastStartContainerCall = time.Now()

	// Start expected containers
	currentRunning, err := listContainers(false)
	if err != nil {
		log.Errorf("Unable to list running containers: %s", err)
		return
	}
	runningNames := []string{}
	for _, v := range currentRunning {
		for _, n := range v.Names {
			runningNames = append(runningNames, strings.Trim(n, "/"))
		}
	}
	for _, n := range expectedRunning {
		if !str.StringInSlice(n, runningNames) {
			bootContainer(n, (*cfg)[n])
		}
	}
}

func cleanContainers() {
	runningContainers, err := listContainers(true)
	if err != nil {
		log.Errorf("Unable to list running containers: %s", err)
		return
	}

	for _, v := range runningContainers {
		_, isManaged := v.Labels[labelIsManaged]

		if strings.HasPrefix(v.Status, "Exited") || strings.HasPrefix(v.Status, "Dead") || (strings.HasPrefix(v.Status, "Created") && isManaged) {
			log.Debugf("Removing container %s (Status %s)", v.ID, v.Status)
			if err := dockerClient.RemoveContainer(docker.RemoveContainerOptions{
				ID:    v.ID,
				Force: true,
			}); err != nil {
				log.Errorf("Unable to remove container %s (Status %s): %s", v.ID, v.Status, err)
			}
		}
	}
}

func parseMounts(mountIn []string) (volumes map[string]struct{}, binds []string) {
	volumes = make(map[string]struct{})
	for _, m := range mountIn {
		if len(m) == 0 {
			continue
		}

		parts := strings.Split(m, ":")
		if len(parts) != 2 && len(parts) != 3 {
			log.Errorf("Invalid default mount: %s", m)
			continue
		}

		binds = append(binds, m)
		volumes[parts[1]] = struct{}{}
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bufio"
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/fsouza/go-dockerclient"

	c "github.com/wantedly/risu/cache"
	"github.com/wantedly/risu/schema"
)

const (
	DefaultDockerEndpoint = "unix:///var/run/docker.sock"
)

var dockerClient *docker.Client

func dockerBuild(build schema.Build) error {
	clonePath := DefaultSourceBaseDir + build.SourceRepo

	if err := addCacheToSrcRepo(build, clonePath); err != nil {
		return err
	}

	client, err := getDockerClient()

	if err != nil {
		return err
	}

	logsReader, outputbuf := io.Pipe()
	go flushLogs(logsReader, build)

	opts := docker.BuildImageOptions{
		Name:                build.ImageName,
		NoCache:             false,
		SuppressOutput:      false,
		RmTmpContainer:      true,
		ForceRmTmpContainer: true,
		Dockerfile:          build.Dockerfile,
		OutputStream:        outputbuf,
		ContextDir:          clonePath,
	}

	if err = client.BuildImage(opts); err != nil {
		return err
	}

	return nil
}

func extractCache(build schema.Build) (string, error) {
	client, err := getDockerClient()

	if err != nil {
		return "", err
	}

	container, err := runContainer(client, build)

	if err != nil {
		return "", err
	}

	defer disposeContainer(client, container)

	saveBaseDir := c.DefaultInflatedCacheDir + getCacheKey(build.SourceRepo) + "/"

	// make directory
	if _, err := os.Stat(c.DefaultInflatedCacheDir); err != nil {
		if err = os.MkdirAll(c.DefaultInflatedCacheDir, 0755); err != nil {
			return "", err
		}
	}

	for _, cacheDirectory := range build.CacheDirectories {
		outputbuf := bytes.NewBuffer(nil)

		if err = client.CopyFromContainer(
			docker.CopyFromContainerOptions{
				Container:    container.ID,
				OutputStream: outputbuf,
				Resource:     cacheDirectory["container"],
			}); err != nil {
			return "", err
		}

		reader := tar.NewReader(outputbuf)

		for {
			header, err := reader.Next()

			if err == io.EOF {
				break
			}

			if err != nil {
				return "", err
			}

			buffer := new(bytes.Buffer)
			outPath := filepath.Join(saveBaseDir, filepath.Dir(cacheDirectory["source"]), header.Name)

			switch header.Typeflag {
			case tar.TypeDir:
				if _, err = os.Stat(outPath); err != nil {
					os.MkdirAll(outPath, 0755)
				}

			case tar.TypeReg, tar.TypeRegA:
				if _, err = io.Copy(buffer, reader); err != nil {
					return "", err
				}

				if err = ioutil.WriteFile(outPath, buffer.Bytes(), os.FileMode(header.Mode)); err != nil {
					return "", err
				}
			}
		}
	}

	return saveBaseDir, nil
}

func dockerPush(build schema.Build) error {
	var dockerImageName, dockerImageTag, dockerRegistry string

	client, err := getDockerClient()

	if err != nil {
		return err
	}

	idx := strings.LastIndex(build.ImageName, ":")

	if idx < 0 {
		dockerRegistry = strings.Split(build.ImageName, "/")[0]
		dockerImageName = strings.Join(strings.Split(build.ImageName, "/")[1:], "/")
		dockerImageTag = "latest"
	} else {
		dockerRegistry = strings.Split(build.ImageName[:idx], "/")[0]
		dockerImageName = strings.Join(strings.Split(build.ImageName[:idx], "/")[1:], "/")
		dockerImageTag = build.ImageName[idx+1:]
	}

	logsReader, outputbuf := io.Pipe()
	go flushLogs(logsReader, build)

	pushOpts := docker.PushImageOptions{
		Name:         dockerImageName,
		Tag:          dockerImageTag,
		Registry:     dockerRegistry,
		OutputStream: outputbuf,
	}
	authConfig := docker.AuthConfiguration{
		Username:      os.Getenv("DOCKER_AUTH_USER_NAME"),
		Password:      os.Getenv("DOCKER_AUTH_USER_PASSWORD"),
		Email:         os.Getenv("DOCKER_AUTH_USER_EMAIL"),
		ServerAddress: dockerRegistry,
	}
	if err := client.PushImage(pushOpts, authConfig); err != nil {
		return err
	}
	return nil
}

func addCacheToSrcRepo(build schema.Build, clonePath string) error {
	cache := c.NewCache(os.Getenv("CACHE_BACKEND"))
	inflatedCachePath, err := cache.Get(getCacheKey(build.SourceRepo))

	if err != nil {
		return err
	}

	for _, cacheDirectory := range build.CacheDirectories {
		cachePath := inflatedCachePath + string(filepath.Separator) + cacheDirectory["source"]
		sourcePath := clonePath + string(filepath.Separator) + cacheDirectory["source"]

		if inflatedCachePath != "" {
			if _, err := os.Stat(sourcePath); err == nil {
				if e := os.RemoveAll(sourcePath); e != nil {
					return e
				}
			}

			if err := os.Rename(cachePath, sourcePath); err != nil {
				return err
			}
		} else {
			if _, err := os.Stat(sourcePath); err != nil {
				os.MkdirAll(sourcePath, 0755)
			}
		}
	}

	return nil
}

func putCache(build schema.Build, cacheSavedDirectory string) error {
	cache := c.NewCache(os.Getenv("CACHE_BACKEND"))

	if err := cache.Put(getCacheKey(build.SourceRepo), cacheSavedDirectory); err != nil {
		return err
	}

	return nil
}

func getCacheKey(text string) string {
	hasher := md5.New()
	hasher.Write([]byte(text))

	return hex.EncodeToString(hasher.Sum(nil))[0:12]
}

func runContainer(client *docker.Client, build schema.Build) (*docker.Container, error) {
	container, err := client.CreateContainer(
		docker.CreateContainerOptions{
			Config: &docker.Config{
				Hostname:        "",
				Domainname:      "",
				User:            "",
				AttachStdin:     false,
				AttachStdout:    false,
				AttachStderr:    false,
				Tty:             false,
				OpenStdin:       false,
				StdinOnce:       false,
				Env:             nil,
				Cmd:             []string{"sleep", "3600"},
				Entrypoint:      []string{"/bin/sh"},
				Image:           build.ImageName,
				Labels:          nil,
				Volumes:         nil,
				WorkingDir:      "",
				NetworkDisabled: false,
				MacAddress:      "",
				ExposedPorts:    nil,
			},
			HostConfig: &docker.HostConfig{},
		})

	if err != nil {
		return nil, err
	}

	if err = client.StartContainer(container.ID, &docker.HostConfig{}); err != nil {
		return nil, err
	}

	return container, nil
}

func disposeContainer(client *docker.Client, container *docker.Container) error {
	if err := client.StopContainer(container.ID, 1); err != nil {
		return err
	}

	return client.RemoveContainer(
		docker.RemoveContainerOptions{
			ID:            container.ID,
			RemoveVolumes: false,
			Force:         true,
		})
}

func getDockerClient() (*docker.Client, error) {
	if dockerClient != nil {
		return dockerClient, nil
	}

	dockerEndpoint := os.Getenv("DOCKER_HOST")

	if dockerEndpoint == "" {
		dockerEndpoint = DefaultDockerEndpoint
	}

	dockerClient, err := docker.NewClient(dockerEndpoint)
	if err != nil {
		return nil, err
	}

	return dockerClient, nil
}

func flushLogs(logsReader io.Reader, build schema.Build) {
	scanner := bufio.NewScanner(logsReader)
	for scanner.Scan() {
		printLog(build, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		printLog(build, "There was an error with the scanner in attached container")
	}
}
<commit_msg>Parse 'localhost:5000/hoge/fuga' correctly<commit_after>package main

import (
	"archive/tar"
	"bufio"
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/fsouza/go-dockerclient"

	c "github.com/wantedly/risu/cache"
	"github.com/wantedly/risu/schema"
)

const (
	DefaultDockerEndpoint = "unix:///var/run/docker.sock"
)

var dockerClient *docker.Client

func dockerBuild(build schema.Build) error {
	clonePath := DefaultSourceBaseDir + build.SourceRepo

	if err := addCacheToSrcRepo(build, clonePath); err != nil {
		return err
	}

	client, err := getDockerClient()

	if err != nil {
		return err
	}

	logsReader, outputbuf := io.Pipe()
	go flushLogs(logsReader, build)

	opts := docker.BuildImageOptions{
		Name:                build.ImageName,
		NoCache:             false,
		SuppressOutput:      false,
		RmTmpContainer:      true,
		ForceRmTmpContainer: true,
		Dockerfile:          build.Dockerfile,
		OutputStream:        outputbuf,
		ContextDir:          clonePath,
	}

	if err = client.BuildImage(opts); err != nil {
		return err
	}

	return nil
}

func extractCache(build schema.Build) (string, error) {
	client, err := getDockerClient()

	if err != nil {
		return "", err
	}

	container, err := runContainer(client, build)

	if err != nil {
		return "", err
	}

	defer disposeContainer(client, container)

	saveBaseDir := c.DefaultInflatedCacheDir + getCacheKey(build.SourceRepo) + "/"

	// make directory
	if _, err := os.Stat(c.DefaultInflatedCacheDir); err != nil {
		if err = os.MkdirAll(c.DefaultInflatedCacheDir, 0755); err != nil {
			return "", err
		}
	}

	for _, cacheDirectory := range build.CacheDirectories {
		outputbuf := bytes.NewBuffer(nil)

		if err = client.CopyFromContainer(
			docker.CopyFromContainerOptions{
				Container:    container.ID,
				OutputStream: outputbuf,
				Resource:     cacheDirectory["container"],
			}); err != nil {
			return "", err
		}

		reader := tar.NewReader(outputbuf)

		for {
			header, err := reader.Next()

			if err == io.EOF {
				break
			}

			if err != nil {
				return "", err
			}

			buffer := new(bytes.Buffer)
			outPath := filepath.Join(saveBaseDir, filepath.Dir(cacheDirectory["source"]), header.Name)

			switch header.Typeflag {
			case tar.TypeDir:
				if _, err = os.Stat(outPath); err != nil {
					os.MkdirAll(outPath, 0755)
				}

			case tar.TypeReg, tar.TypeRegA:
				if _, err = io.Copy(buffer, reader); err != nil {
					return "", err
				}

				if err = ioutil.WriteFile(outPath, buffer.Bytes(), os.FileMode(header.Mode)); err != nil {
					return "", err
				}
			}
		}
	}

	return saveBaseDir, nil
}

func dockerPush(build schema.Build) error {
	var dockerImageName, dockerImageTag, dockerRegistry string

	client, err := getDockerClient()

	if err != nil {
		return err
	}

	idx := strings.LastIndex(build.ImageName, ":")

	if idx < 0 {
		dockerRegistry = strings.Split(build.ImageName, "/")[0]
		dockerImageName = strings.Join(strings.Split(build.ImageName, "/")[1:], "/")
		dockerImageTag = "latest"
	} else {
		tag := build.ImageName[idx+1:]

		if !strings.Contains(tag, "/") {
			dockerRegistry = strings.Split(build.ImageName, "/")[0]
			dockerImageName = strings.Join(strings.Split(build.ImageName, "/")[1:], "/")
			dockerImageTag = "latest"
		} else {
			dockerRegistry = strings.Split(build.ImageName[:idx], "/")[0]
			dockerImageName = strings.Join(strings.Split(build.ImageName[:idx], "/")[1:], "/")
			dockerImageTag = tag
		}
	}

	logsReader, outputbuf := io.Pipe()
	go flushLogs(logsReader, build)

	pushOpts := docker.PushImageOptions{
		Name:         dockerImageName,
		Tag:          dockerImageTag,
		Registry:     dockerRegistry,
		OutputStream: outputbuf,
	}
	authConfig := docker.AuthConfiguration{
		Username:      os.Getenv("DOCKER_AUTH_USER_NAME"),
		Password:      os.Getenv("DOCKER_AUTH_USER_PASSWORD"),
		Email:         os.Getenv("DOCKER_AUTH_USER_EMAIL"),
		ServerAddress: dockerRegistry,
	}
	if err := client.PushImage(pushOpts, authConfig); err != nil {
		return err
	}
	return nil
}

func addCacheToSrcRepo(build schema.Build, clonePath string) error {
	cache := c.NewCache(os.Getenv("CACHE_BACKEND"))
	inflatedCachePath, err := cache.Get(getCacheKey(build.SourceRepo))

	if err != nil {
		return err
	}

	for _, cacheDirectory := range build.CacheDirectories {
		cachePath := inflatedCachePath + string(filepath.Separator) + cacheDirectory["source"]
		sourcePath := clonePath + string(filepath.Separator) + cacheDirectory["source"]

		if inflatedCachePath != "" {
			if _, err := os.Stat(sourcePath); err == nil {
				if e := os.RemoveAll(sourcePath); e != nil {
					return e
				}
			}

			if err := os.Rename(cachePath, sourcePath); err != nil {
				return err
			}
		} else {
			if _, err := os.Stat(sourcePath); err != nil {
				os.MkdirAll(sourcePath, 0755)
			}
		}
	}

	return nil
}

func putCache(build schema.Build, cacheSavedDirectory string) error {
	cache := c.NewCache(os.Getenv("CACHE_BACKEND"))

	if err := cache.Put(getCacheKey(build.SourceRepo), cacheSavedDirectory); err != nil {
		return err
	}

	return nil
}

func getCacheKey(text string) string {
	hasher := md5.New()
	hasher.Write([]byte(text))

	return hex.EncodeToString(hasher.Sum(nil))[0:12]
}

func runContainer(client *docker.Client, build schema.Build) (*docker.Container, error) {
	container, err := client.CreateContainer(
		docker.CreateContainerOptions{
			Config: &docker.Config{
				Hostname:        "",
				Domainname:      "",
				User:            "",
				AttachStdin:     false,
				AttachStdout:    false,
				AttachStderr:    false,
				Tty:             false,
				OpenStdin:       false,
				StdinOnce:       false,
				Env:             nil,
				Cmd:             []string{"sleep", "3600"},
				Entrypoint:      []string{"/bin/sh"},
				Image:           build.ImageName,
				Labels:          nil,
				Volumes:         nil,
				WorkingDir:      "",
				NetworkDisabled: false,
				MacAddress:      "",
				ExposedPorts:    nil,
			},
			HostConfig: &docker.HostConfig{},
		})

	if err != nil {
		return nil, err
	}

	if err = client.StartContainer(container.ID, &docker.HostConfig{}); err != nil {
		return nil, err
	}

	return container, nil
}

func disposeContainer(client *docker.Client, container *docker.Container) error {
	if err := client.StopContainer(container.ID, 1); err != nil {
		return err
	}

	return client.RemoveContainer(
		docker.RemoveContainerOptions{
			ID:            container.ID,
			RemoveVolumes: false,
			Force:         true,
		})
}

func getDockerClient() (*docker.Client, error) {
	if dockerClient != nil {
		return dockerClient, nil
	}

	dockerEndpoint := os.Getenv("DOCKER_HOST")

	if dockerEndpoint == "" {
		dockerEndpoint = DefaultDockerEndpoint
	}

	dockerClient, err := docker.NewClient(dockerEndpoint)
	if err != nil {
		return nil, err
	}

	return dockerClient, nil
}

func flushLogs(logsReader io.Reader, build schema.Build) {
	scanner := bufio.NewScanner(logsReader)
	for scanner.Scan() {
		printLog(build, scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		printLog(build, "There was an error with the scanner in attached container")
	}
}
<|endoftext|>


<commit_before>package dht

import (
	"context"
	"errors"
	"fmt"
	"time"

	proto "github.com/gogo/protobuf/proto"
	cid "github.com/ipfs/go-cid"
	ds "github.com/ipfs/go-datastore"
	u "github.com/ipfs/go-ipfs-util"
	pb "github.com/libp2p/go-libp2p-kad-dht/pb"
	lgbl "github.com/libp2p/go-libp2p-loggables"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	recpb "github.com/libp2p/go-libp2p-record/pb"
	base32 "github.com/whyrusleeping/base32"
)

// The number of closer peers to send on requests.
var CloserPeerCount = KValue

// dhthandler specifies the signature of functions that handle DHT messages.
type dhtHandler func(context.Context, peer.ID, *pb.Message) (*pb.Message, error)

func (dht *IpfsDHT) handlerForMsgType(t pb.Message_MessageType) dhtHandler {
	switch t {
	case pb.Message_GET_VALUE:
		return dht.handleGetValue
	case pb.Message_PUT_VALUE:
		return dht.handlePutValue
	case pb.Message_FIND_NODE:
		return dht.handleFindPeer
	case pb.Message_ADD_PROVIDER:
		return dht.handleAddProvider
	case pb.Message_GET_PROVIDERS:
		return dht.handleGetProviders
	case pb.Message_PING:
		return dht.handlePing
	default:
		return nil
	}
}

func (dht *IpfsDHT) handleGetValue(ctx context.Context, p peer.ID, pmes *pb.Message) (_ *pb.Message, err error) {
	eip := log.EventBegin(ctx, "handleGetValue", p)
	defer func() {
		if err != nil {
			eip.SetError(err)
		}
		eip.Done()
	}()
	log.Debugf("%s handleGetValue for key: %s", dht.self, pmes.GetKey())

	// setup response
	resp := pb.NewMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel())

	// first, is there even a key?
	k := pmes.GetKey()
	if k == "" {
		return nil, errors.New("handleGetValue but no key was provided")
		// TODO: send back an error response? could be bad, but the other node's hanging.
	}

	rec, err := dht.checkLocalDatastore(k)
	if err != nil {
		return nil, err
	}
	resp.Record = rec

	// Find closest peer on given cluster to desired key and reply with that info
	closer := dht.betterPeersToQuery(pmes, p, CloserPeerCount)
	if len(closer) > 0 {
		closerinfos := pstore.PeerInfos(dht.peerstore, closer)
		for _, pi := range closerinfos {
			log.Debugf("handleGetValue returning closer peer: '%s'", pi.ID)
			if len(pi.Addrs) < 1 {
				log.Warningf(`no addresses on peer being sent!
					[local:%s]
					[sending:%s]
					[remote:%s]`, dht.self, pi.ID, p)
			}
		}

		resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), closerinfos)
	}

	return resp, nil
}

func (dht *IpfsDHT) checkLocalDatastore(k string) (*recpb.Record, error) {
	log.Debugf("%s handleGetValue looking into ds", dht.self)
	dskey := convertToDsKey(k)
	iVal, err := dht.datastore.Get(dskey)
	log.Debugf("%s handleGetValue looking into ds GOT %v", dht.self, iVal)

	if err == ds.ErrNotFound {
		return nil, nil
	}

	// if we got an unexpected error, bail.
	if err != nil {
		return nil, err
	}

	// if we have the value, send it back
	log.Debugf("%s handleGetValue success!", dht.self)

	byts, ok := iVal.([]byte)
	if !ok {
		return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey)
	}

	rec := new(recpb.Record)
	err = proto.Unmarshal(byts, rec)
	if err != nil {
		log.Debug("failed to unmarshal DHT record from datastore")
		return nil, err
	}

	var recordIsBad bool
	recvtime, err := u.ParseRFC3339(rec.GetTimeReceived())
	if err != nil {
		log.Info("either no receive time set on record, or it was invalid: ", err)
		recordIsBad = true
	}

	if time.Now().Sub(recvtime) > MaxRecordAge {
		log.Debug("old record found, tossing.")
		recordIsBad = true
	}

	// NOTE: We do not verify the record here beyond checking these timestamps.
	// we put the burden of checking the records on the requester as checking a record
	// may be computationally expensive

	if recordIsBad {
		err := dht.datastore.Delete(dskey)
		if err != nil {
			log.Error("Failed to delete bad record from datastore: ", err)
		}

		return nil, nil // can treat this as not having the record at all
	}

	return rec, nil
}

// Cleans the record (to avoid storing arbitrary data).
func cleanRecord(rec *recpb.Record) {
	rec.XXX_unrecognized = nil
	rec.TimeReceived = nil
}

// Store a value in this peer local storage
func (dht *IpfsDHT) handlePutValue(ctx context.Context, p peer.ID, pmes *pb.Message) (_ *pb.Message, err error) {
	eip := log.EventBegin(ctx, "handlePutValue", p)
	defer func() {
		if err != nil {
			eip.SetError(err)
		}
		eip.Done()
	}()

	rec := pmes.GetRecord()
	if rec == nil {
		log.Infof("Got nil record from: %s", p.Pretty())
		return nil, errors.New("nil record")
	}

	if pmes.GetKey() != rec.GetKey() {
		return nil, errors.New("put key doesn't match record key")
	}

	cleanRecord(rec)

	// Make sure the record is valid (not expired, valid signature etc)
	if err = dht.Validator.Validate(rec.GetKey(), rec.GetValue()); err != nil {
		log.Warningf("Bad dht record in PUT from: %s. %s", p.Pretty(), err)
		return nil, err
	}

	dskey := convertToDsKey(rec.GetKey())

	// Make sure the new record is "better" than the record we have locally.
	// This prevents a record with for example a lower sequence number from
	// overwriting a record with a higher sequence number.
	existing, err := dht.getRecordFromDatastore(dskey)
	if err != nil {
		return nil, err
	}

	if existing != nil {
		recs := [][]byte{rec.GetValue(), existing.GetValue()}
		i, err := dht.Validator.Select(rec.GetKey(), recs)
		if err != nil {
			log.Warningf("Bad dht record in PUT from %s: %s", p.Pretty(), err)
			return nil, err
		}
		if i != 0 {
			log.Infof("DHT record in PUT from %s is older than existing record. Ignoring", p.Pretty())
			return nil, errors.New("old record")
		}
	}

	// record the time we receive every record
	rec.TimeReceived = proto.String(u.FormatRFC3339(time.Now()))

	data, err := proto.Marshal(rec)
	if err != nil {
		return nil, err
	}

	err = dht.datastore.Put(dskey, data)
	log.Debugf("%s handlePutValue %v", dht.self, dskey)
	return pmes, err
}

func (dht *IpfsDHT) getRecordFromDatastore(dskey ds.Key) (*recpb.Record, error) {
	reci, err := dht.datastore.Get(dskey)
	if err == ds.ErrNotFound {
		return nil, nil
	}
	if err != nil {
		log.Warningf("Got error retrieving record with key %s from datastore: %s", dskey, err)
		return nil, err
	}

	byt, ok := reci.([]byte)
	if !ok {
		// Bad data in datastore, log it but don't return an error, we'll just overwrite it
		log.Warningf("Value stored in datastore with key %s is not []byte", dskey)
		return nil, nil
	}

	rec := new(recpb.Record)
	err = proto.Unmarshal(byt, rec)
	if err != nil {
		// Bad data in datastore, log it but don't return an error, we'll just overwrite it
		log.Warningf("Bad record data stored in datastore with key %s: could not unmarshal record", dskey)
		return nil, nil
	}

	err = dht.Validator.Validate(rec.GetKey(), rec.GetValue())
	if err != nil {
		// Invalid record in datastore, probably expired but don't return an error,
		// we'll just overwrite it
		log.Debugf("Local record verify failed: %s (discarded)", err)
		return nil, nil
	}

	return rec, nil
}

func (dht *IpfsDHT) handlePing(_ context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	log.Debugf("%s Responding to ping from %s!\n", dht.self, p)
	return pmes, nil
}

func (dht *IpfsDHT) handleFindPeer(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	defer log.EventBegin(ctx, "handleFindPeer", p).Done()
	resp := pb.NewMessage(pmes.GetType(), "", pmes.GetClusterLevel())
	var closest []peer.ID

	// if looking for self... special case where we send it on CloserPeers.
	if peer.ID(pmes.GetKey()) == dht.self {
		closest = []peer.ID{dht.self}
	} else {
		closest = dht.betterPeersToQuery(pmes, p, CloserPeerCount)
	}

	if closest == nil {
		log.Infof("%s handleFindPeer %s: could not find anything.", dht.self, p)
		return resp, nil
	}

	closestinfos := pstore.PeerInfos(dht.peerstore, closest)
	// possibly an over-allocation but this array is temporary anyways.
	withAddresses := make([]pstore.PeerInfo, 0, len(closestinfos))
	for _, pi := range closestinfos {
		if len(pi.Addrs) > 0 {
			withAddresses = append(withAddresses, pi)
			log.Debugf("handleFindPeer: sending back '%s'", pi.ID)
		}
	}

	resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), withAddresses)
	return resp, nil
}

func (dht *IpfsDHT) handleGetProviders(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	lm := make(lgbl.DeferredMap)
	lm["peer"] = func() interface{} { return p.Pretty() }
	eip := log.EventBegin(ctx, "handleGetProviders", lm)
	defer eip.Done()

	resp := pb.NewMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel())
	c, err := cid.Cast([]byte(pmes.GetKey()))
	if err != nil {
		eip.SetError(err)
		return nil, err
	}

	lm["key"] = func() interface{} { return c.String() }

	// debug logging niceness.
	reqDesc := fmt.Sprintf("%s handleGetProviders(%s, %s): ", dht.self, p, c)
	log.Debugf("%s begin", reqDesc)
	defer log.Debugf("%s end", reqDesc)

	// check if we have this value, to add ourselves as provider.
	has, err := dht.datastore.Has(convertToDsKey(c.KeyString()))
	if err != nil && err != ds.ErrNotFound {
		log.Debugf("unexpected datastore error: %v\n", err)
		has = false
	}

	// setup providers
	providers := dht.providers.GetProviders(ctx, c)
	if has {
		providers = append(providers, dht.self)
		log.Debugf("%s have the value. added self as provider", reqDesc)
	}

	if providers != nil && len(providers) > 0 {
		infos := pstore.PeerInfos(dht.peerstore, providers)
		resp.ProviderPeers = pb.PeerInfosToPBPeers(dht.host.Network(), infos)
		log.Debugf("%s have %d providers: %s", reqDesc, len(providers), infos)
	}

	// Also send closer peers.
	closer := dht.betterPeersToQuery(pmes, p, CloserPeerCount)
	if closer != nil {
		infos := pstore.PeerInfos(dht.peerstore, closer)
		resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), infos)
		log.Debugf("%s have %d closer peers: %s", reqDesc, len(closer), infos)
	}

	return resp, nil
}

func (dht *IpfsDHT) handleAddProvider(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	lm := make(lgbl.DeferredMap)
	lm["peer"] = func() interface{} { return p.Pretty() }
	eip := log.EventBegin(ctx, "handleAddProvider", lm)
	defer eip.Done()

	c, err := cid.Cast([]byte(pmes.GetKey()))
	if err != nil {
		eip.SetError(err)
		return nil, err
	}

	lm["key"] = func() interface{} { return c.String() }

	log.Debugf("%s adding %s as a provider for '%s'\n", dht.self, p, c)

	// add provider should use the address given in the message
	pinfos := pb.PBPeersToPeerInfos(pmes.GetProviderPeers())
	for _, pi := range pinfos {
		if pi.ID != p {
			// we should ignore this provider record! not from originator.
			// (we should sign them and check signature later...)
			log.Debugf("handleAddProvider received provider %s from %s. Ignore.", pi.ID, p)
			continue
		}

		if len(pi.Addrs) < 1 {
			log.Debugf("%s got no valid addresses for provider %s. Ignore.", dht.self, p)
			continue
		}

		log.Infof("received provider %s for %s (addrs: %s)", p, c, pi.Addrs)
		if pi.ID != dht.self { // don't add own addrs.
			// add the received addresses to our peerstore.
			dht.peerstore.AddAddrs(pi.ID, pi.Addrs, pstore.ProviderAddrTTL)
		}
		dht.providers.AddProvider(ctx, c, p)
	}

	return nil, nil
}

func convertToDsKey(s string) ds.Key {
	return ds.NewKey(base32.RawStdEncoding.EncodeToString([]byte(s)))
}
<commit_msg>upgrade local datastore warnings to errors<commit_after>package dht

import (
	"context"
	"errors"
	"fmt"
	"time"

	proto "github.com/gogo/protobuf/proto"
	cid "github.com/ipfs/go-cid"
	ds "github.com/ipfs/go-datastore"
	u "github.com/ipfs/go-ipfs-util"
	pb "github.com/libp2p/go-libp2p-kad-dht/pb"
	lgbl "github.com/libp2p/go-libp2p-loggables"
	peer "github.com/libp2p/go-libp2p-peer"
	pstore "github.com/libp2p/go-libp2p-peerstore"
	recpb "github.com/libp2p/go-libp2p-record/pb"
	base32 "github.com/whyrusleeping/base32"
)

// The number of closer peers to send on requests.
var CloserPeerCount = KValue

// dhthandler specifies the signature of functions that handle DHT messages.
type dhtHandler func(context.Context, peer.ID, *pb.Message) (*pb.Message, error)

func (dht *IpfsDHT) handlerForMsgType(t pb.Message_MessageType) dhtHandler {
	switch t {
	case pb.Message_GET_VALUE:
		return dht.handleGetValue
	case pb.Message_PUT_VALUE:
		return dht.handlePutValue
	case pb.Message_FIND_NODE:
		return dht.handleFindPeer
	case pb.Message_ADD_PROVIDER:
		return dht.handleAddProvider
	case pb.Message_GET_PROVIDERS:
		return dht.handleGetProviders
	case pb.Message_PING:
		return dht.handlePing
	default:
		return nil
	}
}

func (dht *IpfsDHT) handleGetValue(ctx context.Context, p peer.ID, pmes *pb.Message) (_ *pb.Message, err error) {
	eip := log.EventBegin(ctx, "handleGetValue", p)
	defer func() {
		if err != nil {
			eip.SetError(err)
		}
		eip.Done()
	}()
	log.Debugf("%s handleGetValue for key: %s", dht.self, pmes.GetKey())

	// setup response
	resp := pb.NewMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel())

	// first, is there even a key?
	k := pmes.GetKey()
	if k == "" {
		return nil, errors.New("handleGetValue but no key was provided")
		// TODO: send back an error response? could be bad, but the other node's hanging.
	}

	rec, err := dht.checkLocalDatastore(k)
	if err != nil {
		return nil, err
	}
	resp.Record = rec

	// Find closest peer on given cluster to desired key and reply with that info
	closer := dht.betterPeersToQuery(pmes, p, CloserPeerCount)
	if len(closer) > 0 {
		closerinfos := pstore.PeerInfos(dht.peerstore, closer)
		for _, pi := range closerinfos {
			log.Debugf("handleGetValue returning closer peer: '%s'", pi.ID)
			if len(pi.Addrs) < 1 {
				log.Warningf(`no addresses on peer being sent!
					[local:%s]
					[sending:%s]
					[remote:%s]`, dht.self, pi.ID, p)
			}
		}

		resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), closerinfos)
	}

	return resp, nil
}

func (dht *IpfsDHT) checkLocalDatastore(k string) (*recpb.Record, error) {
	log.Debugf("%s handleGetValue looking into ds", dht.self)
	dskey := convertToDsKey(k)
	iVal, err := dht.datastore.Get(dskey)
	log.Debugf("%s handleGetValue looking into ds GOT %v", dht.self, iVal)

	if err == ds.ErrNotFound {
		return nil, nil
	}

	// if we got an unexpected error, bail.
	if err != nil {
		return nil, err
	}

	// if we have the value, send it back
	log.Debugf("%s handleGetValue success!", dht.self)

	byts, ok := iVal.([]byte)
	if !ok {
		return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey)
	}

	rec := new(recpb.Record)
	err = proto.Unmarshal(byts, rec)
	if err != nil {
		log.Debug("failed to unmarshal DHT record from datastore")
		return nil, err
	}

	var recordIsBad bool
	recvtime, err := u.ParseRFC3339(rec.GetTimeReceived())
	if err != nil {
		log.Info("either no receive time set on record, or it was invalid: ", err)
		recordIsBad = true
	}

	if time.Now().Sub(recvtime) > MaxRecordAge {
		log.Debug("old record found, tossing.")
		recordIsBad = true
	}

	// NOTE: We do not verify the record here beyond checking these timestamps.
	// we put the burden of checking the records on the requester as checking a record
	// may be computationally expensive

	if recordIsBad {
		err := dht.datastore.Delete(dskey)
		if err != nil {
			log.Error("Failed to delete bad record from datastore: ", err)
		}

		return nil, nil // can treat this as not having the record at all
	}

	return rec, nil
}

// Cleans the record (to avoid storing arbitrary data).
func cleanRecord(rec *recpb.Record) {
	rec.XXX_unrecognized = nil
	rec.TimeReceived = nil
}

// Store a value in this peer local storage
func (dht *IpfsDHT) handlePutValue(ctx context.Context, p peer.ID, pmes *pb.Message) (_ *pb.Message, err error) {
	eip := log.EventBegin(ctx, "handlePutValue", p)
	defer func() {
		if err != nil {
			eip.SetError(err)
		}
		eip.Done()
	}()

	rec := pmes.GetRecord()
	if rec == nil {
		log.Infof("Got nil record from: %s", p.Pretty())
		return nil, errors.New("nil record")
	}

	if pmes.GetKey() != rec.GetKey() {
		return nil, errors.New("put key doesn't match record key")
	}

	cleanRecord(rec)

	// Make sure the record is valid (not expired, valid signature etc)
	if err = dht.Validator.Validate(rec.GetKey(), rec.GetValue()); err != nil {
		log.Warningf("Bad dht record in PUT from: %s. %s", p.Pretty(), err)
		return nil, err
	}

	dskey := convertToDsKey(rec.GetKey())

	// Make sure the new record is "better" than the record we have locally.
	// This prevents a record with for example a lower sequence number from
	// overwriting a record with a higher sequence number.
	existing, err := dht.getRecordFromDatastore(dskey)
	if err != nil {
		return nil, err
	}

	if existing != nil {
		recs := [][]byte{rec.GetValue(), existing.GetValue()}
		i, err := dht.Validator.Select(rec.GetKey(), recs)
		if err != nil {
			log.Warningf("Bad dht record in PUT from %s: %s", p.Pretty(), err)
			return nil, err
		}
		if i != 0 {
			log.Infof("DHT record in PUT from %s is older than existing record. Ignoring", p.Pretty())
			return nil, errors.New("old record")
		}
	}

	// record the time we receive every record
	rec.TimeReceived = proto.String(u.FormatRFC3339(time.Now()))

	data, err := proto.Marshal(rec)
	if err != nil {
		return nil, err
	}

	err = dht.datastore.Put(dskey, data)
	log.Debugf("%s handlePutValue %v", dht.self, dskey)
	return pmes, err
}

func (dht *IpfsDHT) getRecordFromDatastore(dskey ds.Key) (*recpb.Record, error) {
	reci, err := dht.datastore.Get(dskey)
	if err == ds.ErrNotFound {
		return nil, nil
	}
	if err != nil {
		log.Errorf("Got error retrieving record with key %s from datastore: %s", dskey, err)
		return nil, err
	}

	byt, ok := reci.([]byte)
	if !ok {
		// Bad data in datastore, log it but don't return an error, we'll just overwrite it
		log.Errorf("Value stored in datastore with key %s is not []byte", dskey)
		return nil, nil
	}

	rec := new(recpb.Record)
	err = proto.Unmarshal(byt, rec)
	if err != nil {
		// Bad data in datastore, log it but don't return an error, we'll just overwrite it
		log.Errorf("Bad record data stored in datastore with key %s: could not unmarshal record", dskey)
		return nil, nil
	}

	err = dht.Validator.Validate(rec.GetKey(), rec.GetValue())
	if err != nil {
		// Invalid record in datastore, probably expired but don't return an error,
		// we'll just overwrite it
		log.Debugf("Local record verify failed: %s (discarded)", err)
		return nil, nil
	}

	return rec, nil
}

func (dht *IpfsDHT) handlePing(_ context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	log.Debugf("%s Responding to ping from %s!\n", dht.self, p)
	return pmes, nil
}

func (dht *IpfsDHT) handleFindPeer(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	defer log.EventBegin(ctx, "handleFindPeer", p).Done()
	resp := pb.NewMessage(pmes.GetType(), "", pmes.GetClusterLevel())
	var closest []peer.ID

	// if looking for self... special case where we send it on CloserPeers.
	if peer.ID(pmes.GetKey()) == dht.self {
		closest = []peer.ID{dht.self}
	} else {
		closest = dht.betterPeersToQuery(pmes, p, CloserPeerCount)
	}

	if closest == nil {
		log.Infof("%s handleFindPeer %s: could not find anything.", dht.self, p)
		return resp, nil
	}

	closestinfos := pstore.PeerInfos(dht.peerstore, closest)
	// possibly an over-allocation but this array is temporary anyways.
	withAddresses := make([]pstore.PeerInfo, 0, len(closestinfos))
	for _, pi := range closestinfos {
		if len(pi.Addrs) > 0 {
			withAddresses = append(withAddresses, pi)
			log.Debugf("handleFindPeer: sending back '%s'", pi.ID)
		}
	}

	resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), withAddresses)
	return resp, nil
}

func (dht *IpfsDHT) handleGetProviders(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	lm := make(lgbl.DeferredMap)
	lm["peer"] = func() interface{} { return p.Pretty() }
	eip := log.EventBegin(ctx, "handleGetProviders", lm)
	defer eip.Done()

	resp := pb.NewMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel())
	c, err := cid.Cast([]byte(pmes.GetKey()))
	if err != nil {
		eip.SetError(err)
		return nil, err
	}

	lm["key"] = func() interface{} { return c.String() }

	// debug logging niceness.
	reqDesc := fmt.Sprintf("%s handleGetProviders(%s, %s): ", dht.self, p, c)
	log.Debugf("%s begin", reqDesc)
	defer log.Debugf("%s end", reqDesc)

	// check if we have this value, to add ourselves as provider.
	has, err := dht.datastore.Has(convertToDsKey(c.KeyString()))
	if err != nil && err != ds.ErrNotFound {
		log.Debugf("unexpected datastore error: %v\n", err)
		has = false
	}

	// setup providers
	providers := dht.providers.GetProviders(ctx, c)
	if has {
		providers = append(providers, dht.self)
		log.Debugf("%s have the value. added self as provider", reqDesc)
	}

	if providers != nil && len(providers) > 0 {
		infos := pstore.PeerInfos(dht.peerstore, providers)
		resp.ProviderPeers = pb.PeerInfosToPBPeers(dht.host.Network(), infos)
		log.Debugf("%s have %d providers: %s", reqDesc, len(providers), infos)
	}

	// Also send closer peers.
	closer := dht.betterPeersToQuery(pmes, p, CloserPeerCount)
	if closer != nil {
		infos := pstore.PeerInfos(dht.peerstore, closer)
		resp.CloserPeers = pb.PeerInfosToPBPeers(dht.host.Network(), infos)
		log.Debugf("%s have %d closer peers: %s", reqDesc, len(closer), infos)
	}

	return resp, nil
}

func (dht *IpfsDHT) handleAddProvider(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
	lm := make(lgbl.DeferredMap)
	lm["peer"] = func() interface{} { return p.Pretty() }
	eip := log.EventBegin(ctx, "handleAddProvider", lm)
	defer eip.Done()

	c, err := cid.Cast([]byte(pmes.GetKey()))
	if err != nil {
		eip.SetError(err)
		return nil, err
	}

	lm["key"] = func() interface{} { return c.String() }

	log.Debugf("%s adding %s as a provider for '%s'\n", dht.self, p, c)

	// add provider should use the address given in the message
	pinfos := pb.PBPeersToPeerInfos(pmes.GetProviderPeers())
	for _, pi := range pinfos {
		if pi.ID != p {
			// we should ignore this provider record! not from originator.
			// (we should sign them and check signature later...)
			log.Debugf("handleAddProvider received provider %s from %s. Ignore.", pi.ID, p)
			continue
		}

		if len(pi.Addrs) < 1 {
			log.Debugf("%s got no valid addresses for provider %s. Ignore.", dht.self, p)
			continue
		}

		log.Infof("received provider %s for %s (addrs: %s)", p, c, pi.Addrs)
		if pi.ID != dht.self { // don't add own addrs.
			// add the received addresses to our peerstore.
			dht.peerstore.AddAddrs(pi.ID, pi.Addrs, pstore.ProviderAddrTTL)
		}
		dht.providers.AddProvider(ctx, c, p)
	}

	return nil, nil
}

func convertToDsKey(s string) ds.Key {
	return ds.NewKey(base32.RawStdEncoding.EncodeToString([]byte(s)))
}
<|endoftext|>


<commit_before>// Copyright 2016 Vladimir Osintsev <osintsev@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.
// See the COPYING file in the main directory for details.

package tornote

import (
	"database/sql"
	"net/http"

	"github.com/gorilla/mux"
)

// frontPageHandler render home page.
func frontPageHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index.html", nil)
}

// readNoteHandler show warn screen and destroy note.
func readNoteHandler(db *sql.DB) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		vars := mux.Vars(r)
		id := vars["id"]
		// XXX: Get ciphertext from persistent storage by note id.
		// XXX: Decrypt plaintext.
		// c.Text, err = key.DecryptBytes(ciphertext)
		renderTemplate(w, "note.html", id)
	})
}

// saveNoteHandler save secret note to persistent datastore.
func saveNoteHandler(db *sql.DB) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// XXX:
		data := "123213213213"

		//encrypted := r.FormValue("body")

		// XXX: Save key and note id to securecookie and redirect
		renderTemplate(w, "done.html", data)
	})
}
<commit_msg>Http handlers works with database now<commit_after>// Copyright 2016 Vladimir Osintsev <osintsev@gmail.com>
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.
// See the COPYING file in the main directory for details.

package tornote

import (
	"database/sql"
	"net/http"

	"github.com/gorilla/mux"
	_ "github.com/mattn/go-sqlite3"
)

// frontPageHandler render home page.
func frontPageHandler(w http.ResponseWriter, r *http.Request) {
	renderTemplate(w, "index.html", nil)
}

// readNoteHandler print encrypted data for client-side decrypt and destroy note.
func readNoteHandler(db *sql.DB) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var encrypted string
		vars := mux.Vars(r)

		// Get encrypted note or return error
		err := db.QueryRow("SELECT encrypted from notes where id = ?", vars["id"]).Scan(&encrypted)
		switch {
		case err == sql.ErrNoRows:
			http.Error(w, err.Error(), http.StatusNotFound)
		case err != nil:
			http.Error(w, err.Error(), http.StatusInternalServerError)
		}

		// Deferred destroy note
		defer func() {
			db.QueryRow("DELETE from notes where id = ?", vars["id"])
		}()

		// Print encrypted note to user
		renderTemplate(w, "note.html", encrypted)
	})
}

// saveNoteHandler save secret note to persistent datastore and show ID.
func saveNoteHandler(db *sql.DB) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		encrypted := r.FormValue("encrypted")
		result, err := db.Exec("INSERT INTO notes VALUES (encrypted, ?)", encrypted)
		if err != nil {
			http.Error(w, err.Error(), http.StatusBadRequest)
		}

		// XXX: Show encryption key for this note to user
		renderTemplate(w, "done.html", result.LastInsertId())
	})
}
<|endoftext|>


<commit_before>package rest

import (
	"gopkg.in/zatiti/router.v1"
	"net/http"
)

// write creates a http handler for creating or updating a document depending on the action provided
func (s *Service) process(modelFactory *ModelFactory, action string) router.Handler {
	return func(w http.ResponseWriter, r *http.Request) {
		// When a new request comes in we want a new model instance created to handle that request.
		model := modelFactory.New(r, action)
		// Event is the name used to track the transaction,
		event := model.Name + "_" + action
		// Track how long this function take to return.
		if s.Metrics != nil {
			stop := s.Metrics.NewTimer(event)
			defer func() {
				stop()
				err := s.Metrics.Incr(event, 1)
				if err != nil {
					s.Logger.Error(err)
				}
			}()
		}
		// Send response back to client when this function returns
		defer func() {
			// Get a pointer to the response struct
			response := model.Context.Response
			status := response.Status
			body, err := model.Encode(response.Body)
			if err != nil {
				body = []byte(http.StatusText(http.StatusInternalServerError))
			}
			// Set response headers
			for key, value := range response.Headers {
				w.Header().Set(key, value)
			}
			// Write the response status code
			w.WriteHeader(status)
			// Write the response body
			w.Write(body)
		}()
		var err error
		err = model.Decode()
		if err != nil {
			s.Logger.Error(err)
			return
		}
		// Validate user input
		err = model.Validate()
		if err != nil {
			s.Logger.Error(err.Error())
			return
		}
		// Call the relevant model action
		switch {
		case action == "insert_one":
			err = model.InsertOne()
		case action == "insert_many":
			err = model.InsertMany()
		case action == "update":
			err = model.Update()
		case action == "upsert":
			err = model.Upsert()
		case action == "find_one":
			err = model.FindOne()
		case action == "find_many":
			err = model.FindMany()
		case action == "remove":
			err = model.Remove()
		}
		// Handle failed database operation
		if err != nil {
			s.Logger.Error(err)
		}
		if s.Broker != nil {
			// If event broker is defined send the event to through the stream
			err := s.Broker.Publish(event, &Event{Request: r, Response: &model.Context.Response})
			if err != nil {
				model.Context.Response.Status = http.StatusInternalServerError
				s.Logger.Error(err)
			}
		}
	}
}

// Insert creates a http handler that will create a document in model's database.
func (s *Service) InsertOne(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "insert_one")
}

// Insert creates a http handler that will create a document in model's database.
func (s *Service) InsertMany(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "insert_many")
}

// Update creates a http handler that will updates a document by the model's update selector in model's database
func (s *Service) Update(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "update")
}

// Upsert creates a http handler that will upsert(create or update if it exists) a document selected by the model's upsert selector
func (s *Service) Upsert(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "upsert")
}

// FindOne - creates a http handler that will return one document from a model's database if the id exists
func (s *Service) FindOne(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "find_one")
}

// FindMany - creates a http handler that will list documents from a model's database
func (s *Service) FindMany(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "find_many")
}

// Remove creates a http handler that will delete a document by remove selector specified in the model model
func (s *Service) Remove(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "remove")
}
<commit_msg>refactored process to reduce complexity<commit_after>package rest

import (
	"gopkg.in/zatiti/router.v1"
	"net/http"
)

// write creates a http handler for creating or updating a document depending on the action provided
func (s *Service) process(modelFactory *ModelFactory, action string) router.Handler {
	return func(w http.ResponseWriter, r *http.Request) {
		// When a new request comes in we want a new model instance created to handle that request.
		model := modelFactory.New(r, action)
		// Event is the name used to track the transaction,
		event := model.Name + "_" + action
		// Track how long this function take to return.
		if s.Metrics != nil {
			stop := s.Metrics.NewTimer(event)
			defer func() {
				stop()
				err := s.Metrics.Incr(event, 1)
				if err != nil {
					s.Logger.Error(err)
				}
			}()
		}
		// Send response back to client when this function returns
		defer func() {
			// Get a pointer to the response struct
			response := model.Context.Response
			status := response.Status
			body, err := model.Encode(response.Body)
			if err != nil {
				body = []byte(http.StatusText(http.StatusInternalServerError))
			}
			// Set response headers
			for key, value := range response.Headers {
				w.Header().Set(key, value)
			}
			// Write the response status code
			w.WriteHeader(status)
			// Write the response body
			w.Write(body)
		}()
		var err error
		err = model.Decode()
		if err != nil {
			s.Logger.Error(err)
			return
		}
		// Validate user input
		err = model.Validate()
		if err != nil {
			s.Logger.Error(err.Error())
			return
		}
		// Call the relevant model action
		s.storageOperation(model, action)
		// Handle failed database operation
		if err != nil {
			s.Logger.Error(err)
		}
		if s.Broker != nil {
			// If event broker is defined send the event to through the stream
			err := s.Broker.Publish(event, &Event{Request: r, Response: &model.Context.Response})
			if err != nil {
				model.Context.Response.Status = http.StatusInternalServerError
				s.Logger.Error(err)
			}
		}
	}
}

// dataOperation -
func (s *Service) storageOperation(model *Model, action string) error {
	switch {
	case action == "insert_one":
		err = model.InsertOne()
	case action == "insert_many":
		err = model.InsertMany()
	case action == "update":
		err = model.Update()
	case action == "upsert":
		err = model.Upsert()
	case action == "find_one":
		err = model.FindOne()
	case action == "find_many":
		err = model.FindMany()
	case action == "remove":
		err = model.Remove()
	}
}

// Insert creates a http handler that will create a document in model's database.
func (s *Service) InsertOne(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "insert_one")
}

// Insert creates a http handler that will create a document in model's database.
func (s *Service) InsertMany(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "insert_many")
}

// Update creates a http handler that will updates a document by the model's update selector in model's database
func (s *Service) Update(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "update")
}

// Upsert creates a http handler that will upsert(create or update if it exists) a document selected by the model's upsert selector
func (s *Service) Upsert(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "upsert")
}

// FindOne - creates a http handler that will return one document from a model's database if the id exists
func (s *Service) FindOne(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "find_one")
}

// FindMany - creates a http handler that will list documents from a model's database
func (s *Service) FindMany(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "find_many")
}

// Remove creates a http handler that will delete a document by remove selector specified in the model model
func (s *Service) Remove(modelFactory *ModelFactory) router.Handler {
	return s.process(modelFactory, "remove")
}
<|endoftext|>