<commit_before>package filter
// Copyright ©2011 Dan Kortschak <dan.kortschak@adelaide.edu.au>
//
//   This program is free software: you can redistribute it and/or modify
//   it under the terms of the GNU General Public License as published by
//   the Free Software Foundation, either version 3 of the License, or
//   (at your option) any later version.
//
//   This program is distributed in the hope that it will be useful,
//   but WITHOUT ANY WARRANTY; without even the implied warranty of
//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//   GNU General Public License for more details.
//
//   You should have received a copy of the GNU General Public License
//   along with this program.  If not, see <http://www.gnu.org/licenses/>.
//
import (
	"bio/seq"
	"bio/index/kmerindex"
	"sort"
)

const (
	diagonalPadding = 2
)

type Merger struct { // these are static so should be part of object Merger
	target, query              *seq.Seq
	filterParams               *Params
	leftPadding, bottomPadding int
	binWidth                   int
	selfComparison             bool
	freeTraps, trapList        *Trapezoid
	trapOrder, tail            *Trapezoid
	eoTerm                     *Trapezoid
	trapCount                  int
}

func NewMerger(index *kmerindex.Index, query *seq.Seq, filterParams *Params, selfCompare bool) (m *Merger) {
	tubeWidth := filterParams.TubeOffset + filterParams.MaxError
	binWidth := tubeWidth - 1
	leftPadding := diagonalPadding + binWidth

	eoTerm := &Trapezoid{
		Left:   query.Len() + 1 + leftPadding,
		Right:  query.Len() + 1,
		Bottom: -1,
		Top:    query.Len() + 1,
		Next:   nil,
	}

	m = &Merger{
		target:        index.Seq,
		filterParams:  filterParams,
		query:         query,
		selfComparison: selfCompare,
		bottomPadding: index.GetK() + 2,
		leftPadding:   leftPadding,
		binWidth:      binWidth,
		eoTerm:        eoTerm,
		trapOrder:     eoTerm,
	}

	return m
}

func (self *Merger) MergeFilterHit(hit *FilterHit) {
	Left := -hit.DiagIndex
	if self.selfComparison && Left <= self.filterParams.MaxError {
		return
	}
	Top := hit.QTo
	Bottom := hit.QFrom

	var temp, free *Trapezoid
	for base := self.trapOrder; true; base = temp {
		temp = base.Next
		if Bottom-self.bottomPadding > base.Top {
			if free == nil {
				self.trapOrder = temp
			} else {
				free.Join(temp)
			}
			self.trapList = base.Join(self.trapList)
			self.trapCount++
		} else if Left-diagonalPadding > base.Right {
			free = base
		} else if Left+self.leftPadding >= base.Left {
			if Left+self.binWidth > base.Right {
				base.Right = Left + self.binWidth
			}
			if Left < base.Left {
				base.Left = Left
			}
			if Top > base.Top {
				base.Top = Top
			}

			if free != nil && free.Right+diagonalPadding >= base.Left {
				free.Right = base.Right
				if free.Bottom > base.Bottom {
					free.Bottom = base.Bottom
				}
				if free.Top < base.Top {
					free.Top = base.Top
				}

				free.Join(temp)
				self.freeTraps = base.Join(self.freeTraps)
			} else if temp != nil && temp.Left-diagonalPadding <= base.Right {
				base.Right = temp.Right
				if base.Bottom > temp.Bottom {
					base.Bottom = temp.Bottom
				}
				if base.Top < temp.Top {
					base.Top = temp.Top
				}
				base.Join(temp.Next)
				self.freeTraps = temp.Join(self.freeTraps)
				temp = base.Next
			}

			return
		} else {
			if self.freeTraps == nil {
				self.freeTraps = &Trapezoid{}
			}
			if free == nil {
				self.trapOrder = self.freeTraps
			} else {
				free.Join(self.freeTraps)
			}

			free, self.freeTraps = self.freeTraps.Decapitate()
			free.Join(base)

			free.Top = Top
			free.Bottom = Bottom
			free.Left = Left
			free.Right = Left + self.binWidth

			return
		}
	}
}

func (self *Merger) clipVertical() {
	for base := self.trapList; base != nil; base = base.Next {
		lagPosition := base.Bottom - MaxIGap + 1
		if lagPosition < 0 {
			lagPosition = 0
		}
		lastPosition := base.Top + MaxIGap
		if lastPosition > self.query.Len() {
			lastPosition = self.query.Len()
		}

		i := 0
		for i = lagPosition; i < lastPosition; i++ {
			if lookUp.ValueToCode[self.query.Seq[i]] >= 0 {
				if i-lagPosition >= MaxIGap {
					if lagPosition-base.Bottom > 0 {
						if self.freeTraps == nil {
							self.freeTraps = &Trapezoid{}
						}

						self.freeTraps = self.freeTraps.Shunt(base)

						base.Top = lagPosition
						base = base.Next
						base.Bottom = i
						self.trapCount++
					} else {
						base.Bottom = i
					}
				}
				lagPosition = i + 1
			}
		}
		if i-lagPosition >= MaxIGap {
			base.Top = lagPosition
		}
	}
}

func (self *Merger) clipTrapezoids() {
	for base := self.trapList; base != nil; base = base.Next {
		if base.Top-base.Bottom < self.bottomPadding-2 {
			continue
		}

		aBottom := base.Bottom - base.Right
		aTop := base.Top - base.Left

		lagPosition := aBottom - MaxIGap + 1
		if lagPosition < 0 {
			lagPosition = 0
		}
		lastPosition := aTop + MaxIGap
		if lastPosition > self.target.Len() {
			lastPosition = self.target.Len()
		}

		lagClip := aBottom
		i := 0
		for i = lagPosition; i < lastPosition; i++ {
			if lookUp.ValueToCode[self.target.Seq[i]] >= 0 {
				if i-lagPosition >= MaxIGap {
					if lagPosition > lagClip {
						if self.freeTraps == nil {
							self.freeTraps = &Trapezoid{}
						}

						self.freeTraps = self.freeTraps.Shunt(base)

						base.Clip(lagPosition, lagClip)

						base = base.Next
						self.trapCount++
					}
					lagClip = i
				}
				lagPosition = i + 1
			}
		}

		if i-lagPosition < MaxIGap {
			lagPosition = aTop
		}

		base.Clip(lagPosition, lagClip)

		self.tail = base
	}
}

func (self *Merger) FinaliseMerge() (trapezoids Trapezoids) {
	var next *Trapezoid
	for base := self.trapOrder; base != self.eoTerm; base = next {
		next = base.Next
		self.trapList = base.Join(self.trapList)
		self.trapCount++
	}

	self.clipVertical()
	self.clipTrapezoids()

	if self.tail != nil {
		self.freeTraps = self.tail.Join(self.freeTraps)
	}

	trapezoids = make(Trapezoids, self.trapCount)
	for i, z := 0, self.trapList; i < self.trapCount; i++ {
		trapezoids[i] = z
		z = z.Next
		trapezoids[i].Next = nil
	}

	sort.Sort(Trapezoids(trapezoids))

	return
}

func SumTrapLengths(trapezoids Trapezoids) (sum int) {
	for _, temp := range trapezoids {
		length := temp.Top - temp.Bottom
		sum += length
	}
	return sum
}
<commit_msg>Trivial aesthetic change<commit_after>package filter
// Copyright ©2011 Dan Kortschak <dan.kortschak@adelaide.edu.au>
//
//   This program is free software: you can redistribute it and/or modify
//   it under the terms of the GNU General Public License as published by
//   the Free Software Foundation, either version 3 of the License, or
//   (at your option) any later version.
//
//   This program is distributed in the hope that it will be useful,
//   but WITHOUT ANY WARRANTY; without even the implied warranty of
//   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//   GNU General Public License for more details.
//
//   You should have received a copy of the GNU General Public License
//   along with this program.  If not, see <http://www.gnu.org/licenses/>.
//
import (
	"bio/seq"
	"bio/index/kmerindex"
	"sort"
)

const (
	diagonalPadding = 2
)

type Merger struct { // these are static so should be part of object Merger
	target, query              *seq.Seq
	filterParams               *Params
	leftPadding, bottomPadding int
	binWidth                   int
	selfComparison             bool
	freeTraps, trapList        *Trapezoid
	trapOrder, tail            *Trapezoid
	eoTerm                     *Trapezoid
	trapCount                  int
}

func NewMerger(index *kmerindex.Index, query *seq.Seq, filterParams *Params, selfCompare bool) (m *Merger) {
	tubeWidth := filterParams.TubeOffset + filterParams.MaxError
	binWidth := tubeWidth - 1
	leftPadding := diagonalPadding + binWidth

	eoTerm := &Trapezoid{
		Left:   query.Len() + 1 + leftPadding,
		Right:  query.Len() + 1,
		Bottom: -1,
		Top:    query.Len() + 1,
		Next:   nil,
	}

	m = &Merger{
		target:        index.Seq,
		filterParams:  filterParams,
		query:         query,
		selfComparison: selfCompare,
		bottomPadding: index.GetK() + 2,
		leftPadding:   leftPadding,
		binWidth:      binWidth,
		eoTerm:        eoTerm,
		trapOrder:     eoTerm,
	}

	return m
}

func (self *Merger) MergeFilterHit(hit *FilterHit) {
	Left := -hit.DiagIndex
	if self.selfComparison && Left <= self.filterParams.MaxError {
		return
	}
	Top := hit.QTo
	Bottom := hit.QFrom

	var temp, free *Trapezoid
	for base := self.trapOrder; ; base = temp {
		temp = base.Next
		if Bottom-self.bottomPadding > base.Top {
			if free == nil {
				self.trapOrder = temp
			} else {
				free.Join(temp)
			}
			self.trapList = base.Join(self.trapList)
			self.trapCount++
		} else if Left-diagonalPadding > base.Right {
			free = base
		} else if Left+self.leftPadding >= base.Left {
			if Left+self.binWidth > base.Right {
				base.Right = Left + self.binWidth
			}
			if Left < base.Left {
				base.Left = Left
			}
			if Top > base.Top {
				base.Top = Top
			}

			if free != nil && free.Right+diagonalPadding >= base.Left {
				free.Right = base.Right
				if free.Bottom > base.Bottom {
					free.Bottom = base.Bottom
				}
				if free.Top < base.Top {
					free.Top = base.Top
				}

				free.Join(temp)
				self.freeTraps = base.Join(self.freeTraps)
			} else if temp != nil && temp.Left-diagonalPadding <= base.Right {
				base.Right = temp.Right
				if base.Bottom > temp.Bottom {
					base.Bottom = temp.Bottom
				}
				if base.Top < temp.Top {
					base.Top = temp.Top
				}
				base.Join(temp.Next)
				self.freeTraps = temp.Join(self.freeTraps)
				temp = base.Next
			}

			return
		} else {
			if self.freeTraps == nil {
				self.freeTraps = &Trapezoid{}
			}
			if free == nil {
				self.trapOrder = self.freeTraps
			} else {
				free.Join(self.freeTraps)
			}

			free, self.freeTraps = self.freeTraps.Decapitate()
			free.Join(base)

			free.Top = Top
			free.Bottom = Bottom
			free.Left = Left
			free.Right = Left + self.binWidth

			return
		}
	}
}

func (self *Merger) clipVertical() {
	for base := self.trapList; base != nil; base = base.Next {
		lagPosition := base.Bottom - MaxIGap + 1
		if lagPosition < 0 {
			lagPosition = 0
		}
		lastPosition := base.Top + MaxIGap
		if lastPosition > self.query.Len() {
			lastPosition = self.query.Len()
		}

		i := 0
		for i = lagPosition; i < lastPosition; i++ {
			if lookUp.ValueToCode[self.query.Seq[i]] >= 0 {
				if i-lagPosition >= MaxIGap {
					if lagPosition-base.Bottom > 0 {
						if self.freeTraps == nil {
							self.freeTraps = &Trapezoid{}
						}

						self.freeTraps = self.freeTraps.Shunt(base)

						base.Top = lagPosition
						base = base.Next
						base.Bottom = i
						self.trapCount++
					} else {
						base.Bottom = i
					}
				}
				lagPosition = i + 1
			}
		}
		if i-lagPosition >= MaxIGap {
			base.Top = lagPosition
		}
	}
}

func (self *Merger) clipTrapezoids() {
	for base := self.trapList; base != nil; base = base.Next {
		if base.Top-base.Bottom < self.bottomPadding-2 {
			continue
		}

		aBottom := base.Bottom - base.Right
		aTop := base.Top - base.Left

		lagPosition := aBottom - MaxIGap + 1
		if lagPosition < 0 {
			lagPosition = 0
		}
		lastPosition := aTop + MaxIGap
		if lastPosition > self.target.Len() {
			lastPosition = self.target.Len()
		}

		lagClip := aBottom
		i := 0
		for i = lagPosition; i < lastPosition; i++ {
			if lookUp.ValueToCode[self.target.Seq[i]] >= 0 {
				if i-lagPosition >= MaxIGap {
					if lagPosition > lagClip {
						if self.freeTraps == nil {
							self.freeTraps = &Trapezoid{}
						}

						self.freeTraps = self.freeTraps.Shunt(base)

						base.Clip(lagPosition, lagClip)

						base = base.Next
						self.trapCount++
					}
					lagClip = i
				}
				lagPosition = i + 1
			}
		}

		if i-lagPosition < MaxIGap {
			lagPosition = aTop
		}

		base.Clip(lagPosition, lagClip)

		self.tail = base
	}
}

func (self *Merger) FinaliseMerge() (trapezoids Trapezoids) {
	var next *Trapezoid
	for base := self.trapOrder; base != self.eoTerm; base = next {
		next = base.Next
		self.trapList = base.Join(self.trapList)
		self.trapCount++
	}

	self.clipVertical()
	self.clipTrapezoids()

	if self.tail != nil {
		self.freeTraps = self.tail.Join(self.freeTraps)
	}

	trapezoids = make(Trapezoids, self.trapCount)
	for i, z := 0, self.trapList; i < self.trapCount; i++ {
		trapezoids[i] = z
		z = z.Next
		trapezoids[i].Next = nil
	}

	sort.Sort(Trapezoids(trapezoids))

	return
}

func SumTrapLengths(trapezoids Trapezoids) (sum int) {
	for _, temp := range trapezoids {
		length := temp.Top - temp.Bottom
		sum += length
	}
	return sum
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package armor

import (
	"encoding/base64"
	"io"
	"os"
)

var armorHeaderSep = []byte(": ")
var blockEnd = []byte("\n=")
var newline = []byte("\n")
var armorEndOfLineOut = []byte("-----\n")

// writeSlices writes its arguments to the given Writer.
func writeSlices(out io.Writer, slices ...[]byte) (err os.Error) {
	for _, s := range slices {
		_, err := out.Write(s)
		if err != nil {
			return
		}
	}
	return
}

// lineBreaker breaks data across several lines, all of the same byte length
// (except possibly the last). Lines are broken with a single '\n'.
type lineBreaker struct {
	lineLength  int
	line        []byte
	used        int
	out         io.Writer
	haveWritten bool
}

func newLineBreaker(out io.Writer, lineLength int) *lineBreaker {
	return &lineBreaker{
		lineLength: lineLength,
		line:       make([]byte, lineLength),
		used:       0,
		out:        out,
	}
}

func (l *lineBreaker) Write(b []byte) (n int, err os.Error) {
	n = len(b)

	if n == 0 {
		return
	}

	if l.used == 0 && l.haveWritten {
		_, err = l.out.Write([]byte{'\n'})
		if err != nil {
			return
		}
	}

	if l.used+len(b) < l.lineLength {
		l.used += copy(l.line[l.used:], b)
		return
	}

	l.haveWritten = true
	_, err = l.out.Write(l.line[0:l.used])
	if err != nil {
		return
	}
	excess := l.lineLength - l.used
	l.used = 0

	_, err = l.out.Write(b[0:excess])
	if err != nil {
		return
	}

	_, err = l.Write(b[excess:])
	return
}

func (l *lineBreaker) Close() (err os.Error) {
	if l.used > 0 {
		_, err = l.out.Write(l.line[0:l.used])
		if err != nil {
			return
		}
	}

	return
}

// encoding keeps track of a running CRC24 over the data which has been written
// to it and outputs a OpenPGP checksum when closed, followed by an armor
// trailer.
//
// It's built into a stack of io.Writers:
//    encoding -> base64 encoder -> lineBreaker -> out
type encoding struct {
	out       io.Writer
	breaker   *lineBreaker
	b64       io.WriteCloser
	crc       uint32
	blockType []byte
}

func (e *encoding) Write(data []byte) (n int, err os.Error) {
	e.crc = crc24(e.crc, data)
	return e.b64.Write(data)
}

func (e *encoding) Close() (err os.Error) {
	err = e.b64.Close()
	if err != nil {
		return
	}
	e.breaker.Close()

	var checksumBytes [3]byte
	checksumBytes[0] = byte(e.crc >> 16)
	checksumBytes[1] = byte(e.crc >> 8)
	checksumBytes[2] = byte(e.crc)

	var b64ChecksumBytes [4]byte
	base64.StdEncoding.Encode(b64ChecksumBytes[:], checksumBytes[:])

	return writeSlices(e.out, blockEnd, b64ChecksumBytes[:], newline, armorEnd, e.blockType, armorEndOfLine)
}

// Encode returns a WriteCloser which will encode the data written to it in
// OpenPGP armor.
func Encode(out io.Writer, blockType string, headers map[string]string) (w io.WriteCloser, err os.Error) {
	bType := []byte(blockType)
	err = writeSlices(out, armorStart, bType, armorEndOfLineOut)
	if err != nil {
		return
	}

	for k, v := range headers {
		err = writeSlices(out, []byte(k), armorHeaderSep, []byte(v), newline)
		if err != nil {
			return
		}
	}

	_, err = out.Write(newline)
	if err != nil {
		return
	}

	e := &encoding{
		out:       out,
		breaker:   newLineBreaker(out, 64),
		crc:       crc24Init,
		blockType: bType,
	}
	e.b64 = base64.NewEncoder(base64.StdEncoding, e.breaker)
	return e, nil
}
<commit_msg>openpgp: Fix improper := shadowing<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package armor

import (
	"encoding/base64"
	"io"
	"os"
)

var armorHeaderSep = []byte(": ")
var blockEnd = []byte("\n=")
var newline = []byte("\n")
var armorEndOfLineOut = []byte("-----\n")

// writeSlices writes its arguments to the given Writer.
func writeSlices(out io.Writer, slices ...[]byte) (err os.Error) {
	for _, s := range slices {
		_, err = out.Write(s)
		if err != nil {
			return err
		}
	}
	return
}

// lineBreaker breaks data across several lines, all of the same byte length
// (except possibly the last). Lines are broken with a single '\n'.
type lineBreaker struct {
	lineLength  int
	line        []byte
	used        int
	out         io.Writer
	haveWritten bool
}

func newLineBreaker(out io.Writer, lineLength int) *lineBreaker {
	return &lineBreaker{
		lineLength: lineLength,
		line:       make([]byte, lineLength),
		used:       0,
		out:        out,
	}
}

func (l *lineBreaker) Write(b []byte) (n int, err os.Error) {
	n = len(b)

	if n == 0 {
		return
	}

	if l.used == 0 && l.haveWritten {
		_, err = l.out.Write([]byte{'\n'})
		if err != nil {
			return
		}
	}

	if l.used+len(b) < l.lineLength {
		l.used += copy(l.line[l.used:], b)
		return
	}

	l.haveWritten = true
	_, err = l.out.Write(l.line[0:l.used])
	if err != nil {
		return
	}
	excess := l.lineLength - l.used
	l.used = 0

	_, err = l.out.Write(b[0:excess])
	if err != nil {
		return
	}

	_, err = l.Write(b[excess:])
	return
}

func (l *lineBreaker) Close() (err os.Error) {
	if l.used > 0 {
		_, err = l.out.Write(l.line[0:l.used])
		if err != nil {
			return
		}
	}

	return
}

// encoding keeps track of a running CRC24 over the data which has been written
// to it and outputs a OpenPGP checksum when closed, followed by an armor
// trailer.
//
// It's built into a stack of io.Writers:
//    encoding -> base64 encoder -> lineBreaker -> out
type encoding struct {
	out       io.Writer
	breaker   *lineBreaker
	b64       io.WriteCloser
	crc       uint32
	blockType []byte
}

func (e *encoding) Write(data []byte) (n int, err os.Error) {
	e.crc = crc24(e.crc, data)
	return e.b64.Write(data)
}

func (e *encoding) Close() (err os.Error) {
	err = e.b64.Close()
	if err != nil {
		return
	}
	e.breaker.Close()

	var checksumBytes [3]byte
	checksumBytes[0] = byte(e.crc >> 16)
	checksumBytes[1] = byte(e.crc >> 8)
	checksumBytes[2] = byte(e.crc)

	var b64ChecksumBytes [4]byte
	base64.StdEncoding.Encode(b64ChecksumBytes[:], checksumBytes[:])

	return writeSlices(e.out, blockEnd, b64ChecksumBytes[:], newline, armorEnd, e.blockType, armorEndOfLine)
}

// Encode returns a WriteCloser which will encode the data written to it in
// OpenPGP armor.
func Encode(out io.Writer, blockType string, headers map[string]string) (w io.WriteCloser, err os.Error) {
	bType := []byte(blockType)
	err = writeSlices(out, armorStart, bType, armorEndOfLineOut)
	if err != nil {
		return
	}

	for k, v := range headers {
		err = writeSlices(out, []byte(k), armorHeaderSep, []byte(v), newline)
		if err != nil {
			return
		}
	}

	_, err = out.Write(newline)
	if err != nil {
		return
	}

	e := &encoding{
		out:       out,
		breaker:   newLineBreaker(out, 64),
		crc:       crc24Init,
		blockType: bType,
	}
	e.b64 = base64.NewEncoder(base64.StdEncoding, e.breaker)
	return e, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	flag "github.com/bborbe/flagenv"
	"github.com/bborbe/lock"
	"github.com/golang/glog"
	"os"
	"runtime"
	"time"
)

const (
	defaultWait          = time.Minute * 5
	defaultLockName      = "/var/run/backup_rsync_client.lock"
	parameterConfigPath  = "config"
	parameterTarget      = "target"
	parameterUser        = "user"
	parameterHost        = "host"
	parameterPort        = "port"
	parameterDirectory   = "dir"
	parameterExcludeFrom = "exclude_from"
	parameterWait        = "wait"
	parameterOneTime     = "one-time"
	parameterLock        = "lock"
)

var (
	configPathPtr  = flag.String(parameterConfigPath, "", "path to json config")
	targetPtr      = flag.String(parameterTarget, "", "target")
	userPtr        = flag.String(parameterUser, "", "user")
	hostPtr        = flag.String(parameterHost, "", "host")
	portPtr        = flag.Int(parameterPort, 22, "port")
	dirPtr         = flag.String(parameterDirectory, "", "dir")
	excludeFromPtr = flag.String(parameterExcludeFrom, "", "exclude from")
	waitPtr        = flag.Duration(parameterWait, defaultWait, "wait")
	oneTimePtr     = flag.Bool(parameterOneTime, false, "exit after first fetch")
	lockPtr        = flag.String(parameterLock, defaultLockName, "lock")
)

func main() {
	defer glog.Flush()
	glog.CopyStandardLogTo("info")
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())

	err := do()
	if err != nil {
		glog.Exit(err)
	}
}

func do() error {
	glog.V(2).Infof("do started")
	l := lock.NewLock(*lockPtr)
	if err := l.Lock(); err != nil {
		return err
	}
	defer l.Unlock()

	if err := setSshKeyPermission(); err != nil {
		glog.V(2).Infof("set ssh key permission failed: %v", err)
		return err
	}

	for {
		glog.V(1).Infof("backup started")
		if err := backup(); err != nil {
			return err
		}
		glog.V(1).Infof("backup finished")

		if *oneTimePtr {
			glog.V(2).Infof("one-time => exit")
			return nil
		}

		glog.V(2).Infof("wait %v", *waitPtr)
		time.Sleep(*waitPtr)
		glog.V(2).Infof("sleep done")
	}
	glog.V(2).Infof("do finished")
	return nil
}

func backup() error {
	hosts, err := getHosts()
	if err != nil {
		return err
	}
	if err := validateHosts(hosts); err != nil {
		return err
	}
	targetDirectory, err := getTargetDirectory()
	if err != nil {
		return err
	}
	if err := backupHosts(hosts, *targetDirectory); err != nil {
		return err
	}
	return nil
}

func getTargetDirectory() (*targetDirectory, error) {
	target := targetDirectory(*targetPtr)
	if err := target.IsValid(); err != nil {
		return nil, err
	}
	return &target, nil
}

func backupHosts(hosts []host, targetDirectory targetDirectory) error {
	for _, host := range hosts {
		glog.V(1).Infof("backup %s started", host.Host)
		if err := host.Backup(targetDirectory); err != nil {
			return err
		}
		glog.V(1).Infof("backup %s finished", host.Host)
	}
	return nil
}

func validateHosts(hosts []host) error {
	if len(hosts) == 0 {
		return fmt.Errorf("no host found in config")
	}
	for _, host := range hosts {
		if err := host.Validate(); err != nil {
			return err
		}
	}
	return nil
}

func getHosts() ([]host, error) {
	glog.V(2).Infof("get hosts")
	if configPathPtr != nil {
		configPath := configPath(*configPathPtr)
		if configPath.IsValue() {
			glog.V(2).Infof("read config %s", configPath)
			return configPath.ParseHosts()
		}
	}
	glog.V(2).Infof("create hosts from args")
	return []host{{
		Active:      true,
		User:        *userPtr,
		Host:        *hostPtr,
		Port:        *portPtr,
		Directory:   *dirPtr,
		ExcludeFrom: *excludeFromPtr,
	}}, nil
}

func setSshKeyPermission() error {
	return os.Chmod("/root/.ssh/id_rsa", 0600)
}
<commit_msg>remove set permission<commit_after>package main

import (
	"fmt"
	flag "github.com/bborbe/flagenv"
	"github.com/bborbe/lock"
	"github.com/golang/glog"
	"os"
	"runtime"
	"time"
)

const (
	defaultWait          = time.Minute * 5
	defaultLockName      = "/var/run/backup_rsync_client.lock"
	parameterConfigPath  = "config"
	parameterTarget      = "target"
	parameterUser        = "user"
	parameterHost        = "host"
	parameterPort        = "port"
	parameterDirectory   = "dir"
	parameterExcludeFrom = "exclude_from"
	parameterWait        = "wait"
	parameterOneTime     = "one-time"
	parameterLock        = "lock"
)

var (
	configPathPtr  = flag.String(parameterConfigPath, "", "path to json config")
	targetPtr      = flag.String(parameterTarget, "", "target")
	userPtr        = flag.String(parameterUser, "", "user")
	hostPtr        = flag.String(parameterHost, "", "host")
	portPtr        = flag.Int(parameterPort, 22, "port")
	dirPtr         = flag.String(parameterDirectory, "", "dir")
	excludeFromPtr = flag.String(parameterExcludeFrom, "", "exclude from")
	waitPtr        = flag.Duration(parameterWait, defaultWait, "wait")
	oneTimePtr     = flag.Bool(parameterOneTime, false, "exit after first fetch")
	lockPtr        = flag.String(parameterLock, defaultLockName, "lock")
)

func main() {
	defer glog.Flush()
	glog.CopyStandardLogTo("info")
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())

	err := do()
	if err != nil {
		glog.Exit(err)
	}
}

func do() error {
	glog.V(2).Infof("do started")
	l := lock.NewLock(*lockPtr)
	if err := l.Lock(); err != nil {
		return err
	}
	defer l.Unlock()

	for {
		glog.V(1).Infof("backup started")
		if err := backup(); err != nil {
			return err
		}
		glog.V(1).Infof("backup finished")

		if *oneTimePtr {
			glog.V(2).Infof("one-time => exit")
			return nil
		}

		glog.V(2).Infof("wait %v", *waitPtr)
		time.Sleep(*waitPtr)
		glog.V(2).Infof("sleep done")
	}
	glog.V(2).Infof("do finished")
	return nil
}

func backup() error {
	hosts, err := getHosts()
	if err != nil {
		return err
	}
	if err := validateHosts(hosts); err != nil {
		return err
	}
	targetDirectory, err := getTargetDirectory()
	if err != nil {
		return err
	}
	if err := backupHosts(hosts, *targetDirectory); err != nil {
		return err
	}
	return nil
}

func getTargetDirectory() (*targetDirectory, error) {
	target := targetDirectory(*targetPtr)
	if err := target.IsValid(); err != nil {
		return nil, err
	}
	return &target, nil
}

func backupHosts(hosts []host, targetDirectory targetDirectory) error {
	for _, host := range hosts {
		glog.V(1).Infof("backup %s started", host.Host)
		if err := host.Backup(targetDirectory); err != nil {
			return err
		}
		glog.V(1).Infof("backup %s finished", host.Host)
	}
	return nil
}

func validateHosts(hosts []host) error {
	if len(hosts) == 0 {
		return fmt.Errorf("no host found in config")
	}
	for _, host := range hosts {
		if err := host.Validate(); err != nil {
			return err
		}
	}
	return nil
}

func getHosts() ([]host, error) {
	glog.V(2).Infof("get hosts")
	if configPathPtr != nil {
		configPath := configPath(*configPathPtr)
		if configPath.IsValue() {
			glog.V(2).Infof("read config %s", configPath)
			return configPath.ParseHosts()
		}
	}
	glog.V(2).Infof("create hosts from args")
	return []host{{
		Active:      true,
		User:        *userPtr,
		Host:        *hostPtr,
		Port:        *portPtr,
		Directory:   *dirPtr,
		ExcludeFrom: *excludeFromPtr,
	}}, nil
}
<|endoftext|>


<commit_before>package writers

import (
	"fmt"
	"io"
	"strings"
	"text/template"
	"time"

	"github.com/rightscale/rsc/gen"
)

// ClientWriter struct exposes methods to generate the go API client code
type ClientWriter struct {
	headerTmpl   *template.Template
	resourceTmpl *template.Template
}

// Client writer factory
func NewClientWriter() (*ClientWriter, error) {
	funcMap := template.FuncMap{
		"comment":           comment,
		"now":               time.Now,
		"commandLine":       commandLine,
		"parameters":        parameters,
		"paramsInitializer": paramsInitializer,
		"isPointer":         isPointer,
		"blankCondition":    blankCondition,
		"stripStar":         stripStar,
	}
	headerT, err := template.New("header-client").Funcs(funcMap).Parse(headerTmpl)
	if err != nil {
		return nil, err
	}
	resourceT, err := template.New("resource-client").Funcs(funcMap).Parse(resourceTmpl)
	if err != nil {
		return nil, err
	}
	return &ClientWriter{
		headerTmpl:   headerT,
		resourceTmpl: resourceT,
	}, nil
}

// Write header text
func (c *ClientWriter) WriteHeader(pkg string, w io.Writer) error {
	return c.headerTmpl.Execute(w, pkg)
}

// Write resource header
func (c *ClientWriter) WriteResourceHeader(name string, w io.Writer) {
	fmt.Fprintf(w, "/******  %s ******/\n\n", name)
}

// Write separator between resources and data types
func (c *ClientWriter) WriteTypeSectionHeader(w io.Writer) {
	fmt.Fprintln(w, "\n/****** Parameter Data Types ******/\n\n")
}

// Write type declaration for resource action arguments
func (c *ClientWriter) WriteType(o *gen.ObjectDataType, w io.Writer) {
	fields := make([]string, len(o.Fields))
	for i, f := range o.Fields {
		fields[i] = fmt.Sprintf("%s %s `json:\"%s,omitempty\"`", strings.Title(f.VarName),
			f.Signature(), f.Name)
	}
	decl := fmt.Sprintf("type %s struct {\n%s\n}", o.Name,
		strings.Join(fields, "\n\t"))
	fmt.Fprintf(w, "%s\n\n", decl)
}

// Write code for a resource
func (c *ClientWriter) WriteResource(resource *gen.Resource, w io.Writer) error {
	return c.resourceTmpl.Execute(w, resource)
}

/***** Format helpers *****/

// Generate code used to initialize instance of parameter from given parameter values
func paramInitializer(param *gen.Action) {
}

// Inline templates

const headerTmpl = `
//************************************************************************//
//                     RightScale API client
//
{{comment "Generated " (now.Format "Jan 2, 2006 at 3:04pm (PST)")}}
// Command:
{{comment commandLine}}
//
// The content of this file is auto-generated, DO NOT MODIFY
//************************************************************************//

package {{.}}

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"time"

	"github.com/rightscale/rsc/metadata"
	"github.com/rightscale/rsc/rsapi"
)

// Url resolver produces an action URL and HTTP method from its name and a given resource href.
// The algorithm consists of first extracting the variables from the href and then substituing them
// in the action path. If there are more than one action paths then the algorithm picks the one that
// can substitute the most variables.
type UrlResolver string

func (r *UrlResolver) Url(rName, aName string) (*metadata.ActionPath, error) {
	res, ok := GenMetadata[rName]
	if !ok {
		return nil, fmt.Errorf("No resource with name '%s'", rName)
	}
	var action *metadata.Action
	for _, a := range res.Actions {
		if a.Name == aName {
			action = a
			break
		}
	}
	if action == nil {
		return nil, fmt.Errorf("No action with name '%s' on %s", aName, rName)
	}
	vars, err := res.ExtractVariables(string(*r))
	if err != nil {
		return nil, err
	}
	return action.Url(vars)
}

`

const resourceTmpl = `{{$resource := .}}{{define "ActionBody"}}` + actionBodyTmpl + `{{end}}
{{comment .Description}}{{if .Attributes}}
type {{.Name}} struct { {{range .Attributes}}
{{.FieldName}} {{.FieldType}} ` + "`" + `json:"{{.Name}},omitempty"` + "`" + `{{end}}
}
{{end}}
{{if .Actions}}
//===== Locator

// {{.Name}} resource locator, exposes resource actions.
type {{.Name}}Locator struct {
	UrlResolver
	api *{{.ClientName}}
}

// {{.Name}} resource locator factory
func (api *{{.ClientName}}) {{.Name}}Locator(href string) *{{.Name}}Locator {
	return &{{.Name}}Locator{UrlResolver(href), api}
}
//===== Actions
{{end}}{{range .Actions}}{{range .PathPatterns}}
// {{.HttpMethod}} {{.Path}}{{end}}
{{comment .Description}}
func (loc *{{$resource.Name}}Locator) {{.MethodName}}({{parameters .}}){{if .Return}} ({{.Return}},{{end}} error{{if .Return}}){{end}} {
	{{template "ActionBody" . }}
}
{{end}}
`

const actionBodyTmpl = `{{$action := .}}{{if .Return}}var res {{.Return}}
	{{end}}{{range .Params}}{{if and .Mandatory (blankCondition .VarName .Type)}}{{blankCondition .VarName .Type}}
		return {{if $action.Return}}res, {{end}}fmt.Errorf("{{.VarName}} is required")
	}
	{{end}}{{end}}{{/* end range .Params */}}var queryParams rsapi.ApiParams{{paramsInitializer . 1 "queryParams"}}
	var payloadParams rsapi.ApiParams{{paramsInitializer . 2 "payloadParams"}}
	uri, err := loc.Url("{{$action.ResourceName}}", "{{$action.Name}}")
	if err != nil {
		return {{if $action.Return}}res, {{end}}err
	}
	{{if .Return}}resp, {{else}}_, {{end}}err {{if .Return}}:{{end}}= loc.api.Dispatch(uri.HttpMethod, uri.Path, queryParams, payloadParams)
	if err != nil {
		return {{if $action.Return}}res, {{end}}err
	}
	{{if .ReturnLocation}}location := resp.Header.Get("Location")
	if len(location) == 0 {
		return res, fmt.Errorf("Missing location header in response")
	} else {
		return &{{stripStar .Return}}{UrlResolver(location), loc.api}, nil
	}{{else if .Return}}defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return res, err
	}
	err = json.Unmarshal(respBody, {{if not (isPointer .Return)}}&{{end}}res)
	return res, err{{else}}return nil{{end}}`
<commit_msg>Make go vet happy<commit_after>package writers

import (
	"fmt"
	"io"
	"strings"
	"text/template"
	"time"

	"github.com/rightscale/rsc/gen"
)

// ClientWriter struct exposes methods to generate the go API client code
type ClientWriter struct {
	headerTmpl   *template.Template
	resourceTmpl *template.Template
}

// Client writer factory
func NewClientWriter() (*ClientWriter, error) {
	funcMap := template.FuncMap{
		"comment":           comment,
		"now":               time.Now,
		"commandLine":       commandLine,
		"parameters":        parameters,
		"paramsInitializer": paramsInitializer,
		"isPointer":         isPointer,
		"blankCondition":    blankCondition,
		"stripStar":         stripStar,
	}
	headerT, err := template.New("header-client").Funcs(funcMap).Parse(headerTmpl)
	if err != nil {
		return nil, err
	}
	resourceT, err := template.New("resource-client").Funcs(funcMap).Parse(resourceTmpl)
	if err != nil {
		return nil, err
	}
	return &ClientWriter{
		headerTmpl:   headerT,
		resourceTmpl: resourceT,
	}, nil
}

// Write header text
func (c *ClientWriter) WriteHeader(pkg string, w io.Writer) error {
	return c.headerTmpl.Execute(w, pkg)
}

// Write resource header
func (c *ClientWriter) WriteResourceHeader(name string, w io.Writer) {
	fmt.Fprintf(w, "/******  %s ******/\n\n", name)
}

// Write separator between resources and data types
func (c *ClientWriter) WriteTypeSectionHeader(w io.Writer) {
	fmt.Fprintf(w, "\n/****** Parameter Data Types ******/\n\n\n")
}

// Write type declaration for resource action arguments
func (c *ClientWriter) WriteType(o *gen.ObjectDataType, w io.Writer) {
	fields := make([]string, len(o.Fields))
	for i, f := range o.Fields {
		fields[i] = fmt.Sprintf("%s %s `json:\"%s,omitempty\"`", strings.Title(f.VarName),
			f.Signature(), f.Name)
	}
	decl := fmt.Sprintf("type %s struct {\n%s\n}", o.Name,
		strings.Join(fields, "\n\t"))
	fmt.Fprintf(w, "%s\n\n", decl)
}

// Write code for a resource
func (c *ClientWriter) WriteResource(resource *gen.Resource, w io.Writer) error {
	return c.resourceTmpl.Execute(w, resource)
}

/***** Format helpers *****/

// Generate code used to initialize instance of parameter from given parameter values
func paramInitializer(param *gen.Action) {
}

// Inline templates

const headerTmpl = `
//************************************************************************//
//                     RightScale API client
//
{{comment "Generated " (now.Format "Jan 2, 2006 at 3:04pm (PST)")}}
// Command:
{{comment commandLine}}
//
// The content of this file is auto-generated, DO NOT MODIFY
//************************************************************************//

package {{.}}

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"time"

	"github.com/rightscale/rsc/metadata"
	"github.com/rightscale/rsc/rsapi"
)

// Url resolver produces an action URL and HTTP method from its name and a given resource href.
// The algorithm consists of first extracting the variables from the href and then substituing them
// in the action path. If there are more than one action paths then the algorithm picks the one that
// can substitute the most variables.
type UrlResolver string

func (r *UrlResolver) Url(rName, aName string) (*metadata.ActionPath, error) {
	res, ok := GenMetadata[rName]
	if !ok {
		return nil, fmt.Errorf("No resource with name '%s'", rName)
	}
	var action *metadata.Action
	for _, a := range res.Actions {
		if a.Name == aName {
			action = a
			break
		}
	}
	if action == nil {
		return nil, fmt.Errorf("No action with name '%s' on %s", aName, rName)
	}
	vars, err := res.ExtractVariables(string(*r))
	if err != nil {
		return nil, err
	}
	return action.Url(vars)
}

`

const resourceTmpl = `{{$resource := .}}{{define "ActionBody"}}` + actionBodyTmpl + `{{end}}
{{comment .Description}}{{if .Attributes}}
type {{.Name}} struct { {{range .Attributes}}
{{.FieldName}} {{.FieldType}} ` + "`" + `json:"{{.Name}},omitempty"` + "`" + `{{end}}
}
{{end}}
{{if .Actions}}
//===== Locator

// {{.Name}} resource locator, exposes resource actions.
type {{.Name}}Locator struct {
	UrlResolver
	api *{{.ClientName}}
}

// {{.Name}} resource locator factory
func (api *{{.ClientName}}) {{.Name}}Locator(href string) *{{.Name}}Locator {
	return &{{.Name}}Locator{UrlResolver(href), api}
}
//===== Actions
{{end}}{{range .Actions}}{{range .PathPatterns}}
// {{.HttpMethod}} {{.Path}}{{end}}
{{comment .Description}}
func (loc *{{$resource.Name}}Locator) {{.MethodName}}({{parameters .}}){{if .Return}} ({{.Return}},{{end}} error{{if .Return}}){{end}} {
	{{template "ActionBody" . }}
}
{{end}}
`

const actionBodyTmpl = `{{$action := .}}{{if .Return}}var res {{.Return}}
	{{end}}{{range .Params}}{{if and .Mandatory (blankCondition .VarName .Type)}}{{blankCondition .VarName .Type}}
		return {{if $action.Return}}res, {{end}}fmt.Errorf("{{.VarName}} is required")
	}
	{{end}}{{end}}{{/* end range .Params */}}var queryParams rsapi.ApiParams{{paramsInitializer . 1 "queryParams"}}
	var payloadParams rsapi.ApiParams{{paramsInitializer . 2 "payloadParams"}}
	uri, err := loc.Url("{{$action.ResourceName}}", "{{$action.Name}}")
	if err != nil {
		return {{if $action.Return}}res, {{end}}err
	}
	{{if .Return}}resp, {{else}}_, {{end}}err {{if .Return}}:{{end}}= loc.api.Dispatch(uri.HttpMethod, uri.Path, queryParams, payloadParams)
	if err != nil {
		return {{if $action.Return}}res, {{end}}err
	}
	{{if .ReturnLocation}}location := resp.Header.Get("Location")
	if len(location) == 0 {
		return res, fmt.Errorf("Missing location header in response")
	} else {
		return &{{stripStar .Return}}{UrlResolver(location), loc.api}, nil
	}{{else if .Return}}defer resp.Body.Close()
	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return res, err
	}
	err = json.Unmarshal(respBody, {{if not (isPointer .Return)}}&{{end}}res)
	return res, err{{else}}return nil{{end}}`
<|endoftext|>


<commit_before>package local

import (
	db_schema "github.com/creativesoftwarefdn/weaviate/database/schema"
)

func (l *LocalSchemaManager) RegisterSchemaUpdateCallback(callback func(updatedSchema db_schema.Schema)) {
	l.callbacks = append(l.callbacks, callback)
}

func (l *LocalSchemaManager) triggerCallbacks() {
	schema := l.GetSchema()

	for _, cb := range l.callbacks {
		cb(schema)
	}
}
<commit_msg>Extend schema to also be able to store state for a connector.<commit_after>package local

import (
	db_schema "github.com/creativesoftwarefdn/weaviate/database/schema"
)

func (l *localSchemaManager) RegisterSchemaUpdateCallback(callback func(updatedSchema db_schema.Schema)) {
	l.callbacks = append(l.callbacks, callback)
}

func (l *localSchemaManager) triggerCallbacks() {
	schema := l.GetSchema()

	for _, cb := range l.callbacks {
		cb(schema)
	}
}
<|endoftext|>


<commit_before>package upload

import (
	"crypto/md5"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// Error Incomplete returned by uploader when loaded non-last chunk.
var Incomplete = errors.New("Incomplete")

// Structure describes the state of the original file.
type OriginalFile struct {
	BaseMime string
	Filepath string
	Filename string
	Size     int64
}

func (ofile *OriginalFile) Ext() string {
	return strings.ToLower(filepath.Ext(ofile.Filename))
}

// Downloading files from the received request.
// The root directory of storage, storage,  used to temporarily store chunks.
// Returns an array of the original files and error.
// If you load a portion of the file, chunk, it will be stored in err error Incomplete,
// and in an array of a single file. File size will fit the current size.
func Process(req *http.Request, storage string) ([]*OriginalFile, error) {
	meta, err := ParseMeta(req)
	if err != nil {
		return nil, err
	}

	body, err := NewBody(req.Header.Get("X-File"), req.Body)
	if err != nil {
		return nil, err
	}
	up := &Uploader{Root: storage, Meta: meta, Body: body}

	files, err := up.SaveFiles()
	if err == Incomplete {
		return files, err
	}
	if err != nil {
		return nil, err
	}

	return files, nil
}

// Upload manager.
type Uploader struct {
	Root string
	Meta *Meta
	Body *Body
}

// Function SaveFiles sequentially loads the original files or chunk's.
func (up *Uploader) SaveFiles() ([]*OriginalFile, error) {
	files := make([]*OriginalFile, 0)
	for {
		ofile, err := up.SaveFile()
		if err == io.EOF {
			break
		}

		if err == Incomplete {
			files = append(files, ofile)
			return files, err
		}

		if err != nil {
			return nil, err
		}

		files = append(files, ofile)
	}

	return files, nil
}

// Function loads one or download the original file chunk.
// Asks for the starting position in the body of the request to read the next file.
// Asks for a temporary file.
// Writes data from the request body into a temporary file.
// Specifies the size of the resulting temporary file.
// If the query specified header Content-Range,
// and the size of the resulting file does not match, it returns an error Incomplete.
// Otherwise, defines the basic mime type, and returns the original file.
func (up *Uploader) SaveFile() (*OriginalFile, error) {
	body, filename, err := up.Reader()
	if err != nil {
		return nil, err
	}

	temp_file, err := up.TempFile()
	if err != nil {
		return nil, err
	}
	defer temp_file.Close()

	if err = up.Write(temp_file, body); err != nil {
		return nil, err
	}

	fi, err := temp_file.Stat()
	if err != nil {
		return nil, err
	}

	ofile := &OriginalFile{Filename: filename, Filepath: temp_file.Name(), Size: fi.Size()}

	if up.Meta.Range != nil && ofile.Size != up.Meta.Range.Size {
		return ofile, Incomplete
	}

	ofile.BaseMime, err = IdentifyMime(ofile.Filepath)
	if err != nil {
		return nil, err
	}

	return ofile, nil
}

// Returns the reader to read the file or chunk of request body and the original file name.
// If the request header Conent-Type is multipart/form-data, returns the next copy part.
// If all of part read the case of binary loading read the request body, an error is returned io.EOF.
func (up *Uploader) Reader() (io.Reader, string, error) {
	if up.Meta.MediaType == "multipart/form-data" {
		if up.Body.MR == nil {
			up.Body.MR = multipart.NewReader(up.Body.Body, up.Meta.Boundary)
		}
		for {
			part, err := up.Body.MR.NextPart()
			if err != nil {
				return nil, "", err
			}
			if part.FormName() == "files[]" {
				return part, part.FileName(), nil
			}
		}
	}

	if up.Body.Available == false {
		return nil, "", io.EOF
	}

	up.Body.Available = false

	return up.Body.Body, up.Meta.Filename, nil
}

// Returns a temporary file to download the file or resume chunk.
func (up *Uploader) TempFile() (*os.File, error) {
	if up.Meta.Range == nil {
		return TempFile()
	}
	return TempFileChunks(up.Meta.Range.Start, up.Root, up.Meta.UploadSid, up.Meta.Filename)
}

// Returns the newly created temporary file.
func TempFile() (*os.File, error) {
	return ioutil.TempFile(os.TempDir(), "pavo")
}

// Returns a temporary file to download chunk.
// To calculate a unique file name used cookie named pavo and the original file name.
// File located in the directory chunks storage root directory.
// Before returning the file pointer is shifted by the value of offset,
// in a situation where the pieces are loaded from the second to the last.
func TempFileChunks(offset int64, storage, upload_sid, user_filename string) (*os.File, error) {
	hasher := md5.New()
	hasher.Write([]byte(upload_sid + user_filename))
	filename := hex.EncodeToString(hasher.Sum(nil))

	path := filepath.Join(storage, "chunks")

	err := os.MkdirAll(path, 0755)
	if err != nil {
		return nil, err
	}

	file, err := os.OpenFile(filepath.Join(path, filename), os.O_CREATE|os.O_WRONLY, 0664)
	if err != nil {
		return nil, err
	}

	if _, err = file.Seek(offset, 0); err != nil {
		return nil, err
	}

	return file, nil
}

// The function writes a temporary file value from reader.
func (up *Uploader) Write(temp_file *os.File, body io.Reader) error {
	var err error
	if up.Meta.Range == nil {
		_, err = io.Copy(temp_file, body)
	} else {
		chunk_size := up.Meta.Range.End - up.Meta.Range.Start + 1
		_, err = io.CopyN(temp_file, body, chunk_size)
	}
	return err
}

// Get base mime type
//		$ file --mime-type pic.jpg
//		pic.jpg: image/jpeg
func IdentifyMime(file string) (string, error) {
	out, err := exec.Command("file", "--mime-type", file).CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("identify: err: %s; detail: %s", err, string(out))
	}

	mime := strings.Split(strings.Split(string(out), ": ")[1], "/")[0]

	return mime, nil
}
<commit_msg>Faster MIME finder<commit_after>package upload

import (
	"crypto/md5"
	"encoding/hex"
	"errors"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
)

// Error Incomplete returned by uploader when loaded non-last chunk.
var Incomplete = errors.New("Incomplete")

// Structure describes the state of the original file.
type OriginalFile struct {
	BaseMime string
	Filepath string
	Filename string
	Size     int64
}

func (ofile *OriginalFile) Ext() string {
	return strings.ToLower(filepath.Ext(ofile.Filename))
}

// Downloading files from the received request.
// The root directory of storage, storage,  used to temporarily store chunks.
// Returns an array of the original files and error.
// If you load a portion of the file, chunk, it will be stored in err error Incomplete,
// and in an array of a single file. File size will fit the current size.
func Process(req *http.Request, storage string) ([]*OriginalFile, error) {
	meta, err := ParseMeta(req)
	if err != nil {
		return nil, err
	}

	body, err := NewBody(req.Header.Get("X-File"), req.Body)
	if err != nil {
		return nil, err
	}
	up := &Uploader{Root: storage, Meta: meta, Body: body}

	files, err := up.SaveFiles()
	if err == Incomplete {
		return files, err
	}
	if err != nil {
		return nil, err
	}

	return files, nil
}

// Upload manager.
type Uploader struct {
	Root string
	Meta *Meta
	Body *Body
}

// Function SaveFiles sequentially loads the original files or chunk's.
func (up *Uploader) SaveFiles() ([]*OriginalFile, error) {
	files := make([]*OriginalFile, 0)
	for {
		ofile, err := up.SaveFile()
		if err == io.EOF {
			break
		}

		if err == Incomplete {
			files = append(files, ofile)
			return files, err
		}

		if err != nil {
			return nil, err
		}

		files = append(files, ofile)
	}

	return files, nil
}

// Function loads one or download the original file chunk.
// Asks for the starting position in the body of the request to read the next file.
// Asks for a temporary file.
// Writes data from the request body into a temporary file.
// Specifies the size of the resulting temporary file.
// If the query specified header Content-Range,
// and the size of the resulting file does not match, it returns an error Incomplete.
// Otherwise, defines the basic mime type, and returns the original file.
func (up *Uploader) SaveFile() (*OriginalFile, error) {
	body, filename, err := up.Reader()
	if err != nil {
		return nil, err
	}

	temp_file, err := up.TempFile()
	if err != nil {
		return nil, err
	}
	defer temp_file.Close()

	if err = up.Write(temp_file, body); err != nil {
		return nil, err
	}

	fi, err := temp_file.Stat()
	if err != nil {
		return nil, err
	}

	ofile := &OriginalFile{Filename: filename, Filepath: temp_file.Name(), Size: fi.Size()}

	if up.Meta.Range != nil && ofile.Size != up.Meta.Range.Size {
		return ofile, Incomplete
	}

	ofile.BaseMime, err = identifyMime(ofile.Filepath)
	if err != nil {
		return nil, err
	}

	return ofile, nil
}

// Returns the reader to read the file or chunk of request body and the original file name.
// If the request header Conent-Type is multipart/form-data, returns the next copy part.
// If all of part read the case of binary loading read the request body, an error is returned io.EOF.
func (up *Uploader) Reader() (io.Reader, string, error) {
	if up.Meta.MediaType == "multipart/form-data" {
		if up.Body.MR == nil {
			up.Body.MR = multipart.NewReader(up.Body.Body, up.Meta.Boundary)
		}
		for {
			part, err := up.Body.MR.NextPart()
			if err != nil {
				return nil, "", err
			}
			if part.FormName() == "files[]" {
				return part, part.FileName(), nil
			}
		}
	}

	if up.Body.Available == false {
		return nil, "", io.EOF
	}

	up.Body.Available = false

	return up.Body.Body, up.Meta.Filename, nil
}

// Returns a temporary file to download the file or resume chunk.
func (up *Uploader) TempFile() (*os.File, error) {
	if up.Meta.Range == nil {
		return TempFile()
	}
	return TempFileChunks(up.Meta.Range.Start, up.Root, up.Meta.UploadSid, up.Meta.Filename)
}

// Returns the newly created temporary file.
func TempFile() (*os.File, error) {
	return ioutil.TempFile(os.TempDir(), "pavo")
}

// Returns a temporary file to download chunk.
// To calculate a unique file name used cookie named pavo and the original file name.
// File located in the directory chunks storage root directory.
// Before returning the file pointer is shifted by the value of offset,
// in a situation where the pieces are loaded from the second to the last.
func TempFileChunks(offset int64, storage, upload_sid, user_filename string) (*os.File, error) {
	hasher := md5.New()
	hasher.Write([]byte(upload_sid + user_filename))
	filename := hex.EncodeToString(hasher.Sum(nil))

	path := filepath.Join(storage, "chunks")

	err := os.MkdirAll(path, 0755)
	if err != nil {
		return nil, err
	}

	file, err := os.OpenFile(filepath.Join(path, filename), os.O_CREATE|os.O_WRONLY, 0664)
	if err != nil {
		return nil, err
	}

	if _, err = file.Seek(offset, 0); err != nil {
		return nil, err
	}

	return file, nil
}

// The function writes a temporary file value from reader.
func (up *Uploader) Write(temp_file *os.File, body io.Reader) error {
	var err error
	if up.Meta.Range == nil {
		_, err = io.Copy(temp_file, body)
	} else {
		chunk_size := up.Meta.Range.End - up.Meta.Range.Start + 1
		_, err = io.CopyN(temp_file, body, chunk_size)
	}
	return err
}

// identifyMine gets base mime type.
func identifyMime(file string) (string, error) {
	f, err := os.Open(file)
	if err != nil {
		return "", err
	}
	defer f.Close()
	// DetectContentType reads at most the first 512 bytes
	buf := make([]byte, 512)
	_, err = f.Read(buf)
	if err != nil {
		return "", err
	}
	mime := strings.Split(http.DetectContentType(buf), "/")[0]

	return mime, nil
}
<|endoftext|>


<commit_before>// Package upload encapsulates prepping an image for sending to discord,
// and actually uploading it there.
package upload

import (
	"bytes"
	"encoding/json"
	"fmt"
	"image"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"time"
	"errors"

	"github.com/fogleman/gg"
	"github.com/tardisx/discord-auto-upload/config"
	daulog "github.com/tardisx/discord-auto-upload/log"
	"golang.org/x/image/font/inconsolata"
)

type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

type Uploader struct {
	Uploads []*Upload `json:"uploads"`
}

type Upload struct {
	Uploaded   bool      `json:"uploaded"` // has this file been uploaded to discord
	UploadedAt time.Time `json:"uploaded_at"`

	originalFilename string // path on the local disk
	filenameToUpload string // post-watermark, or just original if unwatermarked

	webhookURL string

	watermark bool // should watermark

	usernameOverride string

	Url string `json:"url"` // url on the discord CDN

	Width  int `json:"width"`
	Height int `json:"height"`

	Client HTTPClient
}

func NewUploader() *Uploader {
	u := Uploader{}
	uploads := make([]*Upload, 0)
	u.Uploads = uploads
	return &u
}

func (u *Uploader) AddFile(file string, conf config.Watcher) {
	thisUpload := Upload{
		Uploaded:         false,
		originalFilename: file,
		watermark:        !conf.NoWatermark,
		webhookURL:       conf.WebHookURL,
		usernameOverride: conf.Username,
	}
	u.Uploads = append(u.Uploads, &thisUpload)
}

// Upload uploads any files that have not yet been uploaded
func (u *Uploader) Upload() {

	for _, upload := range u.Uploads {
		if !upload.Uploaded {
			upload.processUpload()
		}
	}
}

func (u *Upload) processUpload() error {
	if u.webhookURL == "" {
		daulog.SendLog("WebHookURL is not configured - cannot upload!", daulog.LogTypeError)
		return errors.New("webhook url not configured")
	}

	if u.watermark {
		daulog.SendLog("Watermarking", daulog.LogTypeInfo)
		u.applyWatermark()
	} else {
		u.filenameToUpload = u.originalFilename
	}

	extraParams := map[string]string{}

	if u.usernameOverride != "" {
		daulog.SendLog("Overriding username with "+u.usernameOverride, daulog.LogTypeInfo)
		extraParams["username"] = u.usernameOverride
	}

	type DiscordAPIResponseAttachment struct {
		URL      string
		ProxyURL string
		Size     int
		Width    int
		Height   int
		Filename string
	}

	type DiscordAPIResponse struct {
		Attachments []DiscordAPIResponseAttachment
		ID          int64 `json:",string"`
	}

	var retriesRemaining = 5
	for retriesRemaining > 0 {

		request, err := newfileUploadRequest(u.webhookURL, extraParams, "file", u.filenameToUpload)
		if err != nil {
			log.Printf("error creating upload request: %s", err)
			return fmt.Errorf("could not create upload request: %s", err)
		}
		start := time.Now()

		if u.Client == nil {
	          // if no client was specified (a unit test) then create
		  // a default one
		  u.Client = &http.Client{Timeout: time.Second * 30}
		}

		resp, err := u.Client.Do(request)
		if err != nil {
			log.Print("Error performing request:", err)
			retriesRemaining--
			sleepForRetries(retriesRemaining)
			continue
		} else {

			if resp.StatusCode != 200 {
				// {"message": "Request entity too large", "code": 40005}
				log.Print("Bad response from server:", resp.StatusCode)
				if b, err := ioutil.ReadAll(resp.Body); err == nil {
					log.Print("Body:", string(b))
					daulog.SendLog(fmt.Sprintf("Bad response: %s", string(b)), daulog.LogTypeError)
				}
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}

			resBody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				log.Print("could not deal with body: ", err)
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			resp.Body.Close()

			var res DiscordAPIResponse
			err = json.Unmarshal(resBody, &res)

			//  {"id": "851092588608880670", "type": 0, "content": "", "channel_id": "849615269706203171", "author": {"bot": true, "id": "849615314274484224", "username": "abcdedf", "avatar": null, "discriminator": "0000"}, "attachments": [{"id": "851092588332449812", "filename": "dau480457962.png", "size": 859505, "url": "https://cdn.discordapp.com/attachments/849615269706203171/851092588332449812/dau480457962.png", "proxy_url": "https://media.discordapp.net/attachments/849615269706203171/851092588332449812/dau480457962.png", "width": 640, "height": 640, "content_type": "image/png"}], "embeds": [], "mentions": [], "mention_roles": [], "pinned": false, "mention_everyone": false, "tts": false, "timestamp": "2021-06-06T13:38:05.660000+00:00", "edited_timestamp": null, "flags": 0, "components": [], "webhook_id": "849615314274484224"}

			daulog.SendLog(fmt.Sprintf("Response: %s", string(resBody[:])), daulog.LogTypeDebug)

			if err != nil {
				log.Print("could not parse JSON: ", err)
				fmt.Println("Response was:", string(resBody[:]))
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			if len(res.Attachments) < 1 {
				log.Print("bad response - no attachments?")
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			var a = res.Attachments[0]
			elapsed := time.Since(start)
			rate := float64(a.Size) / elapsed.Seconds() / 1024.0

			daulog.SendLog(fmt.Sprintf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height), daulog.LogTypeInfo)
			daulog.SendLog(fmt.Sprintf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate), daulog.LogTypeInfo)

			u.Url = a.URL
			u.Uploaded = true
			u.Width = a.Width
			u.Height = a.Height
			u.UploadedAt = time.Now()

			break
		}
	}

	if u.watermark {
		daulog.SendLog(fmt.Sprintf("Removing temporary file: %s", u.filenameToUpload), daulog.LogTypeDebug)
		os.Remove(u.filenameToUpload)
	}

	if retriesRemaining == 0 {
		daulog.SendLog("Failed to upload, even after all retries", daulog.LogTypeError)
	}
	return nil
}

func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("could not open file '%s': %s", path, err)
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		log.Fatal("Could not copy: ", err)
	}

	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", uri, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	return req, err
}

func (u *Upload) applyWatermark() {

	reader, err := os.Open(u.originalFilename)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	im, _, err := image.Decode(reader)
	if err != nil {
		daulog.SendLog(fmt.Sprintf("Cannot decode image: %v - skipping watermarking", err), daulog.LogTypeError)
		u.watermark = false
		u.filenameToUpload = u.originalFilename
		return
	}
	bounds := im.Bounds()
	// var S float64 = float64(bounds.Max.X)

	dc := gg.NewContext(bounds.Max.X, bounds.Max.Y)
	dc.Clear()
	dc.SetRGB(0, 0, 0)

	dc.SetFontFace(inconsolata.Regular8x16)

	dc.DrawImage(im, 0, 0)

	dc.DrawRoundedRectangle(0, float64(bounds.Max.Y-18.0), 320, float64(bounds.Max.Y), 0)
	dc.SetRGB(0, 0, 0)
	dc.Fill()

	dc.SetRGB(1, 1, 1)

	dc.DrawString("github.com/tardisx/discord-auto-upload", 5.0, float64(bounds.Max.Y)-5.0)

	tempfile, err := ioutil.TempFile("", "dau")
	if err != nil {
		log.Fatal(err)
	}
	tempfile.Close()
	os.Remove(tempfile.Name())
	actualName := tempfile.Name() + ".png"

	dc.SavePNG(actualName)
	u.filenameToUpload = actualName
}

func sleepForRetries(retry int) {
	if retry == 0 {
		return
	}
	retryTime := (6-retry)*(6-retry) + 6
	daulog.SendLog(fmt.Sprintf("Will retry in %d seconds (%d remaining attempts)", retryTime, retry), daulog.LogTypeError)
	time.Sleep(time.Duration(retryTime) * time.Second)
}
<commit_msg>Mark completely failed uploads as failed so we don't keep retrying them<commit_after>// Package upload encapsulates prepping an image for sending to discord,
// and actually uploading it there.
package upload

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"image"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/fogleman/gg"
	"github.com/tardisx/discord-auto-upload/config"
	daulog "github.com/tardisx/discord-auto-upload/log"
	"golang.org/x/image/font/inconsolata"
)

type HTTPClient interface {
	Do(req *http.Request) (*http.Response, error)
}

type Uploader struct {
	Uploads []*Upload `json:"uploads"`
}

type Upload struct {
	Uploaded   bool      `json:"uploaded"` // has this file been uploaded to discord
	UploadedAt time.Time `json:"uploaded_at"`

	Failed bool `json:"failed"`

	originalFilename string // path on the local disk
	filenameToUpload string // post-watermark, or just original if unwatermarked

	webhookURL string

	watermark bool // should watermark

	usernameOverride string

	Url string `json:"url"` // url on the discord CDN

	Width  int `json:"width"`
	Height int `json:"height"`

	Client HTTPClient
}

func NewUploader() *Uploader {
	u := Uploader{}
	uploads := make([]*Upload, 0)
	u.Uploads = uploads
	return &u
}

func (u *Uploader) AddFile(file string, conf config.Watcher) {
	thisUpload := Upload{
		Uploaded:         false,
		originalFilename: file,
		watermark:        !conf.NoWatermark,
		webhookURL:       conf.WebHookURL,
		usernameOverride: conf.Username,
	}
	u.Uploads = append(u.Uploads, &thisUpload)
}

// Upload uploads any files that have not yet been uploaded
func (u *Uploader) Upload() {

	for _, upload := range u.Uploads {
		if !upload.Uploaded && !upload.Failed {
			upload.processUpload()
		}
	}
}

func (u *Upload) processUpload() error {
	daulog.SendLog(fmt.Sprintf("Uploading: %s", u.originalFilename), daulog.LogTypeInfo)

	if u.webhookURL == "" {
		daulog.SendLog("WebHookURL is not configured - cannot upload!", daulog.LogTypeError)
		return errors.New("webhook url not configured")
	}

	if u.watermark {
		daulog.SendLog("Watermarking image", daulog.LogTypeInfo)
		u.applyWatermark()
	} else {
		u.filenameToUpload = u.originalFilename
	}

	extraParams := map[string]string{}

	if u.usernameOverride != "" {
		daulog.SendLog("Overriding username with "+u.usernameOverride, daulog.LogTypeInfo)
		extraParams["username"] = u.usernameOverride
	}

	type DiscordAPIResponseAttachment struct {
		URL      string
		ProxyURL string
		Size     int
		Width    int
		Height   int
		Filename string
	}

	type DiscordAPIResponse struct {
		Attachments []DiscordAPIResponseAttachment
		ID          int64 `json:",string"`
	}

	var retriesRemaining = 5
	for retriesRemaining > 0 {

		request, err := newfileUploadRequest(u.webhookURL, extraParams, "file", u.filenameToUpload)
		if err != nil {
			log.Printf("error creating upload request: %s", err)
			return fmt.Errorf("could not create upload request: %s", err)
		}
		start := time.Now()

		if u.Client == nil {
			// if no client was specified (a unit test) then create
			// a default one
			u.Client = &http.Client{Timeout: time.Second * 30}
		}

		resp, err := u.Client.Do(request)
		if err != nil {
			log.Print("Error performing request:", err)
			retriesRemaining--
			sleepForRetries(retriesRemaining)
			continue
		} else {

			if resp.StatusCode != 200 {
				// {"message": "Request entity too large", "code": 40005}
				log.Print("Bad response from server:", resp.StatusCode)
				if b, err := ioutil.ReadAll(resp.Body); err == nil {
					log.Print("Body:", string(b))
					daulog.SendLog(fmt.Sprintf("Bad response: %s", string(b)), daulog.LogTypeError)
				}
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}

			resBody, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				log.Print("could not deal with body: ", err)
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			resp.Body.Close()

			var res DiscordAPIResponse
			err = json.Unmarshal(resBody, &res)

			//  {"id": "851092588608880670", "type": 0, "content": "", "channel_id": "849615269706203171", "author": {"bot": true, "id": "849615314274484224", "username": "abcdedf", "avatar": null, "discriminator": "0000"}, "attachments": [{"id": "851092588332449812", "filename": "dau480457962.png", "size": 859505, "url": "https://cdn.discordapp.com/attachments/849615269706203171/851092588332449812/dau480457962.png", "proxy_url": "https://media.discordapp.net/attachments/849615269706203171/851092588332449812/dau480457962.png", "width": 640, "height": 640, "content_type": "image/png"}], "embeds": [], "mentions": [], "mention_roles": [], "pinned": false, "mention_everyone": false, "tts": false, "timestamp": "2021-06-06T13:38:05.660000+00:00", "edited_timestamp": null, "flags": 0, "components": [], "webhook_id": "849615314274484224"}

			daulog.SendLog(fmt.Sprintf("Response: %s", string(resBody[:])), daulog.LogTypeDebug)

			if err != nil {
				log.Print("could not parse JSON: ", err)
				fmt.Println("Response was:", string(resBody[:]))
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			if len(res.Attachments) < 1 {
				log.Print("bad response - no attachments?")
				retriesRemaining--
				sleepForRetries(retriesRemaining)
				continue
			}
			var a = res.Attachments[0]
			elapsed := time.Since(start)
			rate := float64(a.Size) / elapsed.Seconds() / 1024.0

			daulog.SendLog(fmt.Sprintf("Uploaded to %s %dx%d", a.URL, a.Width, a.Height), daulog.LogTypeInfo)
			daulog.SendLog(fmt.Sprintf("id: %d, %d bytes transferred in %.2f seconds (%.2f KiB/s)", res.ID, a.Size, elapsed.Seconds(), rate), daulog.LogTypeInfo)

			u.Url = a.URL
			u.Uploaded = true
			u.Width = a.Width
			u.Height = a.Height
			u.UploadedAt = time.Now()

			break
		}
	}

	if u.watermark {
		daulog.SendLog(fmt.Sprintf("Removing temporary file: %s", u.filenameToUpload), daulog.LogTypeDebug)
		os.Remove(u.filenameToUpload)
	}

	if retriesRemaining == 0 {
		daulog.SendLog("Failed to upload, even after all retries", daulog.LogTypeError)
		u.Failed = true
		return errors.New("could not upload after all retries")
	}
	return nil
}

func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, fmt.Errorf("could not open file '%s': %s", path, err)
	}
	defer file.Close()

	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)
	part, err := writer.CreateFormFile(paramName, filepath.Base(path))
	if err != nil {
		return nil, err
	}
	_, err = io.Copy(part, file)
	if err != nil {
		log.Fatal("Could not copy: ", err)
	}

	for key, val := range params {
		_ = writer.WriteField(key, val)
	}
	err = writer.Close()
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest("POST", uri, body)
	req.Header.Set("Content-Type", writer.FormDataContentType())
	return req, err
}

func (u *Upload) applyWatermark() {

	reader, err := os.Open(u.originalFilename)
	if err != nil {
		log.Fatal(err)
	}
	defer reader.Close()

	im, _, err := image.Decode(reader)
	if err != nil {
		daulog.SendLog(fmt.Sprintf("Cannot decode image: %v - skipping watermarking", err), daulog.LogTypeError)
		u.watermark = false
		u.filenameToUpload = u.originalFilename
		return
	}
	bounds := im.Bounds()
	// var S float64 = float64(bounds.Max.X)

	dc := gg.NewContext(bounds.Max.X, bounds.Max.Y)
	dc.Clear()
	dc.SetRGB(0, 0, 0)

	dc.SetFontFace(inconsolata.Regular8x16)

	dc.DrawImage(im, 0, 0)

	dc.DrawRoundedRectangle(0, float64(bounds.Max.Y-18.0), 320, float64(bounds.Max.Y), 0)
	dc.SetRGB(0, 0, 0)
	dc.Fill()

	dc.SetRGB(1, 1, 1)

	dc.DrawString("github.com/tardisx/discord-auto-upload", 5.0, float64(bounds.Max.Y)-5.0)

	tempfile, err := ioutil.TempFile("", "dau")
	if err != nil {
		log.Fatal(err)
	}
	tempfile.Close()
	os.Remove(tempfile.Name())
	actualName := tempfile.Name() + ".png"

	dc.SavePNG(actualName)
	u.filenameToUpload = actualName
}

func sleepForRetries(retry int) {
	if retry == 0 {
		return
	}
	retryTime := (6-retry)*(6-retry) + 6
	daulog.SendLog(fmt.Sprintf("Will retry in %d seconds (%d remaining attempts)", retryTime, retry), daulog.LogTypeError)
	time.Sleep(time.Duration(retryTime) * time.Second)
}
<|endoftext|>


<commit_before>package coreapi

import (
	"context"

	core "github.com/ipfs/go-ipfs/core"
	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	ipfspath "github.com/ipfs/go-ipfs/path"
	uio "github.com/ipfs/go-ipfs/unixfs/io"

	cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid"
)

type CoreAPI struct {
	node *core.IpfsNode
}

// NewCoreAPI creates new instance of IPFS CoreAPI backed by go-ipfs Node
func NewCoreAPI(n *core.IpfsNode) coreiface.CoreAPI {
	api := &CoreAPI{n}
	return api
}

func (api *CoreAPI) Unixfs() coreiface.UnixfsAPI {
	return (*UnixfsAPI)(api)
}

func (api *CoreAPI) Dag() coreiface.DagAPI {
	return &DagAPI{api, nil}
}

func (api *CoreAPI) Name() coreiface.NameAPI {
	return &NameAPI{api, nil}
}

func (api *CoreAPI) Key() coreiface.KeyAPI {
	return &KeyAPI{api, nil}
}

func (api *CoreAPI) ResolveNode(ctx context.Context, p coreiface.Path) (coreiface.Node, error) {
	p, err := api.ResolvePath(ctx, p)
	if err != nil {
		return nil, err
	}

	node, err := api.node.DAG.Get(ctx, p.Cid())
	if err != nil {
		return nil, err
	}
	return node, nil
}

// TODO: store all of ipfspath.Resolver.ResolvePathComponents() in Path
func (api *CoreAPI) ResolvePath(ctx context.Context, p coreiface.Path) (coreiface.Path, error) {
	if p.Resolved() {
		return p, nil
	}

	r := &ipfspath.Resolver{
		DAG:         api.node.DAG,
		ResolveOnce: uio.ResolveUnixfsOnce,
	}

	p2 := ipfspath.FromString(p.String())
	node, err := core.Resolve(ctx, api.node.Namesys, r, p2)
	if err == core.ErrNoNamesys {
		return nil, coreiface.ErrOffline
	} else if err != nil {
		return nil, err
	}

	var root *cid.Cid
	if p2.IsJustAKey() {
		root = node.Cid()
	}

	return ResolvedPath(p.String(), node.Cid(), root), nil
}

// Implements coreiface.Path
type path struct {
	path ipfspath.Path
	cid  *cid.Cid
	root *cid.Cid
}

func ParsePath(p string) (coreiface.Path, error) {
	pp, err := ipfspath.ParsePath(p)
	if err != nil {
		return nil, err
	}
	return &path{path: pp}, nil
}

func ParseCid(c *cid.Cid) coreiface.Path {
	return &path{path: ipfspath.FromCid(c), cid: c, root: c}
}

func ResolvedPath(p string, c *cid.Cid, r *cid.Cid) coreiface.Path {
	return &path{path: ipfspath.FromString(p), cid: c, root: r}
}

func (p *path) String() string { return p.path.String() }
func (p *path) Cid() *cid.Cid  { return p.cid }
func (p *path) Root() *cid.Cid { return p.root }
func (p *path) Resolved() bool { return p.cid != nil }
<commit_msg>docs for coreapi method impls<commit_after>package coreapi

import (
	"context"

	core "github.com/ipfs/go-ipfs/core"
	coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	ipfspath "github.com/ipfs/go-ipfs/path"
	uio "github.com/ipfs/go-ipfs/unixfs/io"

	cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid"
)

type CoreAPI struct {
	node *core.IpfsNode
}

// NewCoreAPI creates new instance of IPFS CoreAPI backed by go-ipfs Node.
func NewCoreAPI(n *core.IpfsNode) coreiface.CoreAPI {
	api := &CoreAPI{n}
	return api
}

// Unixfs returns the UnixfsAPI interface backed by the go-ipfs node
func (api *CoreAPI) Unixfs() coreiface.UnixfsAPI {
	return (*UnixfsAPI)(api)
}

// Dag returns the DagAPI interface backed by the go-ipfs node
func (api *CoreAPI) Dag() coreiface.DagAPI {
	return &DagAPI{api, nil}
}

// Name returns the NameAPI interface backed by the go-ipfs node
func (api *CoreAPI) Name() coreiface.NameAPI {
	return &NameAPI{api, nil}
}

// Key returns the KeyAPI interface backed by the go-ipfs node
func (api *CoreAPI) Key() coreiface.KeyAPI {
	return &KeyAPI{api, nil}
}

// ResolveNode resolves the path `p` using Unixfx resolver, gets and returns the
// resolved Node.
func (api *CoreAPI) ResolveNode(ctx context.Context, p coreiface.Path) (coreiface.Node, error) {
	p, err := api.ResolvePath(ctx, p)
	if err != nil {
		return nil, err
	}

	node, err := api.node.DAG.Get(ctx, p.Cid())
	if err != nil {
		return nil, err
	}
	return node, nil
}

// ResolvePath resolves the path `p` using Unixfs resolver, returns the
// resolved path.
// TODO: store all of ipfspath.Resolver.ResolvePathComponents() in Path
func (api *CoreAPI) ResolvePath(ctx context.Context, p coreiface.Path) (coreiface.Path, error) {
	if p.Resolved() {
		return p, nil
	}

	r := &ipfspath.Resolver{
		DAG:         api.node.DAG,
		ResolveOnce: uio.ResolveUnixfsOnce,
	}

	p2 := ipfspath.FromString(p.String())
	node, err := core.Resolve(ctx, api.node.Namesys, r, p2)
	if err == core.ErrNoNamesys {
		return nil, coreiface.ErrOffline
	} else if err != nil {
		return nil, err
	}

	var root *cid.Cid
	if p2.IsJustAKey() {
		root = node.Cid()
	}

	return ResolvedPath(p.String(), node.Cid(), root), nil
}

// Implements coreiface.Path
type path struct {
	path ipfspath.Path
	cid  *cid.Cid
	root *cid.Cid
}

// ParsePath parses path `p` using ipfspath parser, returns the parsed path.
func ParsePath(p string) (coreiface.Path, error) {
	pp, err := ipfspath.ParsePath(p)
	if err != nil {
		return nil, err
	}
	return &path{path: pp}, nil
}

// ParseCid parses the path from `c`, retruns the parsed path.
func ParseCid(c *cid.Cid) coreiface.Path {
	return &path{path: ipfspath.FromCid(c), cid: c, root: c}
}

// ResolvePath parses path from string `p`, returns parsed path.
func ResolvedPath(p string, c *cid.Cid, r *cid.Cid) coreiface.Path {
	return &path{path: ipfspath.FromString(p), cid: c, root: r}
}

func (p *path) String() string { return p.path.String() }
func (p *path) Cid() *cid.Cid  { return p.cid }
func (p *path) Root() *cid.Cid { return p.root }
func (p *path) Resolved() bool { return p.cid != nil }
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/igneous-systems/logit"

	"github.com/igneous-systems/pickett"
	"github.com/igneous-systems/pickett/io"
)

func contains(s []string, target string) bool {
	for _, candidate := range s {
		if candidate == target {
			return true
		}
	}
	return false
}

func makeIOObjects(path string) (io.Helper, io.DockerCli, io.EtcdClient, io.VirtualBox, error) {
	helper, err := io.NewHelper(path)
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("can't read %s: %v", path, err)
	}
	cli, err := io.NewDockerCli()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to connect to docker server, maybe its not running? %v", err)
	}
	etcd, err := io.NewEtcdClient()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to connect to etcd, maybe its not running? %v", err)
	}
	vbox, err := io.NewVirtualBox()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to run vboxmanage: %v", err)
	}
	return helper, cli, etcd, vbox, nil
}

// trueMain is the entry point of the program with the targets filled in
// and a working helper.
func trueMain(targets []string, helper io.Helper, cli io.DockerCli, etcd io.EtcdClient, vbox io.VirtualBox) error {
	reader := helper.ConfigReader()
	config, err := pickett.NewConfig(reader, helper, cli, etcd, vbox)
	if err != nil {
		return fmt.Errorf("can't understand config file %s: %v", helper.ConfigFile(), err)
	}
	buildables, runnables := config.EntryPoints()
	run := false
	runTarget := ""

	// if you don't tell us what to build, we build everything with no outgoing
	// edges, the "root" of a backchain
	if len(targets) == 0 {
		targets = buildables
	} else {
		//if you do tell us, we need know if it's runnable
		for _, t := range targets {
			if contains(buildables, t) {
				continue
			}
			if contains(runnables, t) {
				if run {
					return fmt.Errorf("can only run one target (%s and %s both runnable)", runTarget, t)
				}
				run = true
				runTarget = t
				continue
			}
			return fmt.Errorf("don't know anything about target %s", t)
		}
	}
	for _, target := range targets {
		if target == runTarget {
			continue
		}
		err := config.Build(target)
		if err != nil {
			return fmt.Errorf("an error occurred while building target '%v': %v", target, err)
		}
	}
	if runTarget != "" {
		err = config.Execute(runTarget)
		if err != nil {
			return fmt.Errorf("an error occurred while running target '%v': %v", runTarget, err)
		}
	}
	return nil
}

var flog = logit.NewNestedLoggerFromCaller(logit.Global)

func main() {
	os.Exit(wrappedMain())
}

// Wrapped to make os.Exit work well with logit
func wrappedMain() int {
	var debug bool
	var configFile string

	flag.BoolVar(&debug, "debug", false, "turns on verbose logging for pickett developers")
	flag.StringVar(&configFile, "config", "Pickett.json", "use a custom pickett configuration file")
	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		usage()
		os.Exit(0)
	}

	var logFilterLvl logit.Level
	if debug {
		logFilterLvl = logit.DEBUG
	} else {
		logFilterLvl = logit.INFO
	}
	logit.Global.ModifyFilterLvl("stdout", logFilterLvl, nil, nil)
	defer logit.Flush(-1)

	wd, err := os.Getwd()
	if err != nil {
		panic("can't get working directory!")
	}

	_, err = os.Open(filepath.Join(wd, configFile))
	if err != nil {
		flog.Errorf("can't find configuration file: %s\n", filepath.Join(wd, configFile))
		return 1
	}

	helper, docker, etcd, vbox, err := makeIOObjects(filepath.Join(wd, configFile))
	if err != nil {
		flog.Errorf("failed to make IO objects: %v", err)
		return 1
	}
	reader := helper.ConfigReader()
	config, err := pickett.NewConfig(reader, helper, docker, etcd, vbox)
	if err != nil {
		flog.Errorf("Can't understand config file %s: %v", err.Error(), helper.ConfigFile())
		return 1
	}
	switch args[0] {
	case "run":
		err = pickett.CmdRun(args[1:], config)
	case "build":
		err = pickett.CmdBuild(args[1:], config)
	case "status":
		err = pickett.CmdStatus(args[1:], config)
	case "stop":
		err = pickett.CmdStop(args[1:], config)
	case "drop":
		err = pickett.CmdDrop(args[1:], config)
	case "wipe":
		err = pickett.CmdWipe(args[1:], config)
	case "help":
		usage()
		return 0
	default:
		usage()
		return 1
	}

	if err != nil {
		flog.Errorf("%s: %v", args[0], err)
		return 1
	}
	return 0
}

func usage() {
	// There doesn't seem to be a better way to mix flags usage with arguments usage ?
	error := fmt.Errorf(`Usage of pickett, expected an action as the first argument, one of:
- run [topology.node]             Runs a specific node in a topology, including all depedencies.
- status [tags or topology.node]  Shows the status of all the known buildable tags and/or runnable nodes.
- build [tags]                    Build all tags or specified tags.
- stop [topology.node]            Stop all or a specific node.
- drop [topology.node]            Stop and delete all or a specific node.
- wipe [tags]                     Delete all or specified tags (forces rebuild next time)
- help                            Print this help message`)
	fmt.Println(error)
}
<commit_msg>Fail early when DOCKER_HOST not set<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/igneous-systems/logit"

	"github.com/igneous-systems/pickett"
	"github.com/igneous-systems/pickett/io"
)

func contains(s []string, target string) bool {
	for _, candidate := range s {
		if candidate == target {
			return true
		}
	}
	return false
}

func makeIOObjects(path string) (io.Helper, io.DockerCli, io.EtcdClient, io.VirtualBox, error) {
	helper, err := io.NewHelper(path)
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("can't read %s: %v", path, err)
	}
	cli, err := io.NewDockerCli()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to connect to docker server, maybe its not running? %v", err)
	}
	etcd, err := io.NewEtcdClient()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to connect to etcd, maybe its not running? %v", err)
	}
	vbox, err := io.NewVirtualBox()
	if err != nil {
		return nil, nil, nil, nil, fmt.Errorf("failed to run vboxmanage: %v", err)
	}
	return helper, cli, etcd, vbox, nil
}

// trueMain is the entry point of the program with the targets filled in
// and a working helper.
func trueMain(targets []string, helper io.Helper, cli io.DockerCli, etcd io.EtcdClient, vbox io.VirtualBox) error {
	reader := helper.ConfigReader()
	config, err := pickett.NewConfig(reader, helper, cli, etcd, vbox)
	if err != nil {
		return fmt.Errorf("can't understand config file %s: %v", helper.ConfigFile(), err)
	}
	buildables, runnables := config.EntryPoints()
	run := false
	runTarget := ""

	// if you don't tell us what to build, we build everything with no outgoing
	// edges, the "root" of a backchain
	if len(targets) == 0 {
		targets = buildables
	} else {
		//if you do tell us, we need know if it's runnable
		for _, t := range targets {
			if contains(buildables, t) {
				continue
			}
			if contains(runnables, t) {
				if run {
					return fmt.Errorf("can only run one target (%s and %s both runnable)", runTarget, t)
				}
				run = true
				runTarget = t
				continue
			}
			return fmt.Errorf("don't know anything about target %s", t)
		}
	}
	for _, target := range targets {
		if target == runTarget {
			continue
		}
		err := config.Build(target)
		if err != nil {
			return fmt.Errorf("an error occurred while building target '%v': %v", target, err)
		}
	}
	if runTarget != "" {
		err = config.Execute(runTarget)
		if err != nil {
			return fmt.Errorf("an error occurred while running target '%v': %v", runTarget, err)
		}
	}
	return nil
}

var flog = logit.NewNestedLoggerFromCaller(logit.Global)

func main() {
	os.Exit(wrappedMain())
}

// Wrapped to make os.Exit work well with logit
func wrappedMain() int {
	var debug bool
	var configFile string

	flag.BoolVar(&debug, "debug", false, "turns on verbose logging for pickett developers")
	flag.StringVar(&configFile, "config", "Pickett.json", "use a custom pickett configuration file")
	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		usage()
		os.Exit(0)
	}

	var logFilterLvl logit.Level
	if debug {
		logFilterLvl = logit.DEBUG
	} else {
		logFilterLvl = logit.INFO
	}
	logit.Global.ModifyFilterLvl("stdout", logFilterLvl, nil, nil)
	defer logit.Flush(-1)

	wd, err := os.Getwd()
	if err != nil {
		panic("can't get working directory!")
	}

	if os.Getenv("DOCKER_HOST") == "" {
		fmt.Fprintf(os.Stderr, "DOCKER_HOST not set; suggest DOCKER_HOST=tcp://:2375 (for local launcher)\n")
		return 1
	}

	_, err = os.Open(filepath.Join(wd, configFile))
	if err != nil {
		flog.Errorf("can't find configuration file: %s\n", filepath.Join(wd, configFile))
		return 1
	}

	helper, docker, etcd, vbox, err := makeIOObjects(filepath.Join(wd, configFile))
	if err != nil {
		flog.Errorf("failed to make IO objects: %v", err)
		return 1
	}
	reader := helper.ConfigReader()
	config, err := pickett.NewConfig(reader, helper, docker, etcd, vbox)
	if err != nil {
		flog.Errorf("Can't understand config file %s: %v", err.Error(), helper.ConfigFile())
		return 1
	}
	switch args[0] {
	case "run":
		err = pickett.CmdRun(args[1:], config)
	case "build":
		err = pickett.CmdBuild(args[1:], config)
	case "status":
		err = pickett.CmdStatus(args[1:], config)
	case "stop":
		err = pickett.CmdStop(args[1:], config)
	case "drop":
		err = pickett.CmdDrop(args[1:], config)
	case "wipe":
		err = pickett.CmdWipe(args[1:], config)
	case "help":
		usage()
		return 0
	default:
		usage()
		return 1
	}

	if err != nil {
		flog.Errorf("%s: %v", args[0], err)
		return 1
	}
	return 0
}

func usage() {
	// There doesn't seem to be a better way to mix flags usage with arguments usage ?
	error := fmt.Errorf(`Usage of pickett, expected an action as the first argument, one of:
- run [topology.node]             Runs a specific node in a topology, including all depedencies.
- status [tags or topology.node]  Shows the status of all the known buildable tags and/or runnable nodes.
- build [tags]                    Build all tags or specified tags.
- stop [topology.node]            Stop all or a specific node.
- drop [topology.node]            Stop and delete all or a specific node.
- wipe [tags]                     Delete all or specified tags (forces rebuild next time)
- help                            Print this help message`)
	fmt.Println(error)
}
<|endoftext|>


<commit_before>package event

import "github.com/libp2p/go-libp2p-core/network"

// EvtPeerStateChange should be emitted everytime we form a connection with a peer or drop our last
// connection with the peer. Essentially, it is emitted in two cases:
// a) We form a/any connection with a peer.
// b) We go from having a connection/s with a peer to having no connection with the peer.
// It contains the connection handle and the new connection state.
type EvtPeerStateChange struct {
	Connection network.Conn
	NewState   network.Connectedness
}
<commit_msg>changes as per review<commit_after>package event

import (
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
)

// EvtPeerConnectednessChange should be emitted every time we form a connection with a peer or drop our last
// connection with the peer. Essentially, it is emitted in two cases:
// a) We form a/any connection with a peer.
// b) We go from having a connection/s with a peer to having no connection with the peer.
// It contains the Id of the remote peer and the new connectedness state.
type EvtPeerConnectednessChange struct {
	RemotePeerId  peer.ID
	Connectedness network.Connectedness
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package api

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/gorilla/mux"
	"github.com/sapcc/limes/pkg/db"
	"github.com/sapcc/limes/pkg/limes"
)

//VersionData is used by version advertisement handlers.
type VersionData struct {
	Status string            `json:"status"`
	ID     string            `json:"id"`
	Links  []VersionLinkData `json:"links"`
}

//VersionLinkData is used by version advertisement handlers, as part of the
//VersionData struct.
type VersionLinkData struct {
	URL      string `json:"href"`
	Relation string `json:"rel"`
	Type     string `json:"type,omitempty"`
}

type v1Provider struct {
	Driver      limes.Driver
	Config      limes.APIConfiguration
	VersionData VersionData
}

//NewV1Router creates a mux.Router that serves the Limes v1 API.
//It also returns the VersionData for this API version which is needed for the
//version advertisement on "GET /".
func NewV1Router(driver limes.Driver, config limes.APIConfiguration) (*mux.Router, VersionData) {
	r := mux.NewRouter()
	p := &v1Provider{
		Driver: driver,
		Config: config,
	}
	p.VersionData = VersionData{
		Status: "EXPERIMENTAL",
		ID:     "v1",
		Links: []VersionLinkData{
			VersionLinkData{
				Relation: "self",
				URL:      p.Path(),
			},
			VersionLinkData{
				Relation: "describedby",
				URL:      "https://github.com/sapcc/limes/tree/master/docs",
				Type:     "text/html",
			},
		},
	}

	r.Methods("GET").Path("/v1/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ReturnJSON(w, 200, map[string]interface{}{"version": p.VersionData})
	})

	r.Methods("GET").Path("/v1/domains").HandlerFunc(p.ListDomains)
	r.Methods("GET").Path("/v1/domains/{domain_id}").HandlerFunc(p.GetDomain)
	r.Methods("POST").Path("/v1/domains/discover").HandlerFunc(p.DiscoverDomains)

	r.Methods("GET").Path("/v1/domains/{domain_id}/projects").HandlerFunc(p.ListProjects)
	r.Methods("GET").Path("/v1/domains/{domain_id}/projects/{project_id}").HandlerFunc(p.GetProject)
	r.Methods("POST").Path("/v1/domains/{domain_id}/projects/discover").HandlerFunc(p.DiscoverProjects)
	r.Methods("POST").Path("/v1/domains/{domain_id}/projects/{project_id}/sync").HandlerFunc(p.SyncProject)

	return r, p.VersionData
}

//ReturnJSON is a convenience function for HTTP handlers returning JSON data.
//The `code` argument specifies the HTTP response code, usually 200.
func ReturnJSON(w http.ResponseWriter, code int, data interface{}) {
	bytes, err := json.Marshal(&data)
	if err == nil {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(code)
		w.Write(bytes)
	} else {
		http.Error(w, err.Error(), 500)
	}
}

//ReturnError produces an error response with HTTP status code 500 if the given
//error is non-nil. Otherwise, nothing is done and false is returned.
func ReturnError(w http.ResponseWriter, err error) bool {
	if err == nil {
		return false
	}

	http.Error(w, err.Error(), 500)
	return true
}

//Path constructs a full URL for a given URL path below the /v1/ endpoint.
func (p *v1Provider) Path(elements ...string) string {
	parts := []string{
		strings.TrimSuffix(p.Driver.Cluster().CatalogURL, "/"),
		"v1",
	}
	parts = append(parts, elements...)
	return strings.Join(parts, "/")
}

//FindDomainFromRequest loads the db.Domain referenced by the :domain_id path
//parameter. Any errors will be written into the response immediately and cause
//a nil return value.
func (p *v1Provider) FindDomainFromRequest(w http.ResponseWriter, r *http.Request) *db.Domain {
	domainUUID := mux.Vars(r)["domain_id"]
	if domainUUID == "" {
		http.Error(w, "domain ID missing", 400)
		return nil
	}

	var domain db.Domain
	err := db.DB.SelectOne(&domain, `SELECT * FROM domains WHERE uuid = $1 AND cluster_id = $2`,
		domainUUID, p.Driver.Cluster().ID,
	)
	switch {
	case err == sql.ErrNoRows:
		http.Error(w, "no such domain (if it was just created, try to POST /domains/discover)", 404)
		return nil
	case ReturnError(w, err):
		return nil
	default:
		return &domain
	}
}

//FindProjectFromRequest loads the db.Project referenced by the :project_id
//path parameter, and verifies that it is located within the given domain.
func (p *v1Provider) FindProjectFromRequest(w http.ResponseWriter, r *http.Request, domain *db.Domain) *db.Project {
	project, ok := p.FindProjectFromRequestIfExists(w, r, domain)
	if ok && project == nil {
		msg := fmt.Sprintf(
			"no such project (if it was just created, try to POST /domains/%s/projects/discover)",
			mux.Vars(r)["domain_id"],
		)
		http.Error(w, msg, 404)
		return nil
	}
	return project
}

//FindProjectFromRequestIfExists works like FindProjectFromRequest, but returns
//a nil project instead of producing an error if the project does not exist in
//the local DB yet.
func (p *v1Provider) FindProjectFromRequestIfExists(w http.ResponseWriter, r *http.Request, domain *db.Domain) (project *db.Project, ok bool) {
	projectUUID := mux.Vars(r)["project_id"]
	if projectUUID == "" {
		http.Error(w, "project ID missing", 400)
		return nil, false
	}

	project = &db.Project{}
	err := db.DB.SelectOne(project, `SELECT * FROM projects WHERE uuid = $1`, projectUUID)
	switch {
	case err == sql.ErrNoRows:
		return nil, true
	case err == nil && domain.ID != project.DomainID:
		http.Error(w, "no such project", 404)
		return nil, false
	case ReturnError(w, err):
		return nil, false
	default:
		return project, true
	}
}
<commit_msg>list missing API calls as a reminder to myself<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package api

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"

	"github.com/gorilla/mux"
	"github.com/sapcc/limes/pkg/db"
	"github.com/sapcc/limes/pkg/limes"
)

//VersionData is used by version advertisement handlers.
type VersionData struct {
	Status string            `json:"status"`
	ID     string            `json:"id"`
	Links  []VersionLinkData `json:"links"`
}

//VersionLinkData is used by version advertisement handlers, as part of the
//VersionData struct.
type VersionLinkData struct {
	URL      string `json:"href"`
	Relation string `json:"rel"`
	Type     string `json:"type,omitempty"`
}

type v1Provider struct {
	Driver      limes.Driver
	Config      limes.APIConfiguration
	VersionData VersionData
}

//NewV1Router creates a mux.Router that serves the Limes v1 API.
//It also returns the VersionData for this API version which is needed for the
//version advertisement on "GET /".
func NewV1Router(driver limes.Driver, config limes.APIConfiguration) (*mux.Router, VersionData) {
	r := mux.NewRouter()
	p := &v1Provider{
		Driver: driver,
		Config: config,
	}
	p.VersionData = VersionData{
		Status: "EXPERIMENTAL",
		ID:     "v1",
		Links: []VersionLinkData{
			VersionLinkData{
				Relation: "self",
				URL:      p.Path(),
			},
			VersionLinkData{
				Relation: "describedby",
				URL:      "https://github.com/sapcc/limes/tree/master/docs",
				Type:     "text/html",
			},
		},
	}

	r.Methods("GET").Path("/v1/").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ReturnJSON(w, 200, map[string]interface{}{"version": p.VersionData})
	})

	// r.Methods("GET").Path("/v1/clusters").HandlerFunc(p.ListClusters)
	// r.Methods("GET").Path("/v1/clusters/{cluster_id}").HandlerFunc(p.GetCluster)

	r.Methods("GET").Path("/v1/domains").HandlerFunc(p.ListDomains)
	r.Methods("GET").Path("/v1/domains/{domain_id}").HandlerFunc(p.GetDomain)
	r.Methods("POST").Path("/v1/domains/discover").HandlerFunc(p.DiscoverDomains)
	// r.Methods("PUT").Path("/v1/domains/{domain_id}").HandlerFunc(p.PutDomain)

	r.Methods("GET").Path("/v1/domains/{domain_id}/projects").HandlerFunc(p.ListProjects)
	r.Methods("GET").Path("/v1/domains/{domain_id}/projects/{project_id}").HandlerFunc(p.GetProject)
	r.Methods("POST").Path("/v1/domains/{domain_id}/projects/discover").HandlerFunc(p.DiscoverProjects)
	r.Methods("POST").Path("/v1/domains/{domain_id}/projects/{project_id}/sync").HandlerFunc(p.SyncProject)
	// r.Methods("PUT").Path("/v1/domains/{domain_id}/projects/{project_id}").HandlerFunc(p.PutProject)

	return r, p.VersionData
}

//ReturnJSON is a convenience function for HTTP handlers returning JSON data.
//The `code` argument specifies the HTTP response code, usually 200.
func ReturnJSON(w http.ResponseWriter, code int, data interface{}) {
	bytes, err := json.Marshal(&data)
	if err == nil {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(code)
		w.Write(bytes)
	} else {
		http.Error(w, err.Error(), 500)
	}
}

//ReturnError produces an error response with HTTP status code 500 if the given
//error is non-nil. Otherwise, nothing is done and false is returned.
func ReturnError(w http.ResponseWriter, err error) bool {
	if err == nil {
		return false
	}

	http.Error(w, err.Error(), 500)
	return true
}

//Path constructs a full URL for a given URL path below the /v1/ endpoint.
func (p *v1Provider) Path(elements ...string) string {
	parts := []string{
		strings.TrimSuffix(p.Driver.Cluster().CatalogURL, "/"),
		"v1",
	}
	parts = append(parts, elements...)
	return strings.Join(parts, "/")
}

//FindDomainFromRequest loads the db.Domain referenced by the :domain_id path
//parameter. Any errors will be written into the response immediately and cause
//a nil return value.
func (p *v1Provider) FindDomainFromRequest(w http.ResponseWriter, r *http.Request) *db.Domain {
	domainUUID := mux.Vars(r)["domain_id"]
	if domainUUID == "" {
		http.Error(w, "domain ID missing", 400)
		return nil
	}

	var domain db.Domain
	err := db.DB.SelectOne(&domain, `SELECT * FROM domains WHERE uuid = $1 AND cluster_id = $2`,
		domainUUID, p.Driver.Cluster().ID,
	)
	switch {
	case err == sql.ErrNoRows:
		http.Error(w, "no such domain (if it was just created, try to POST /domains/discover)", 404)
		return nil
	case ReturnError(w, err):
		return nil
	default:
		return &domain
	}
}

//FindProjectFromRequest loads the db.Project referenced by the :project_id
//path parameter, and verifies that it is located within the given domain.
func (p *v1Provider) FindProjectFromRequest(w http.ResponseWriter, r *http.Request, domain *db.Domain) *db.Project {
	project, ok := p.FindProjectFromRequestIfExists(w, r, domain)
	if ok && project == nil {
		msg := fmt.Sprintf(
			"no such project (if it was just created, try to POST /domains/%s/projects/discover)",
			mux.Vars(r)["domain_id"],
		)
		http.Error(w, msg, 404)
		return nil
	}
	return project
}

//FindProjectFromRequestIfExists works like FindProjectFromRequest, but returns
//a nil project instead of producing an error if the project does not exist in
//the local DB yet.
func (p *v1Provider) FindProjectFromRequestIfExists(w http.ResponseWriter, r *http.Request, domain *db.Domain) (project *db.Project, ok bool) {
	projectUUID := mux.Vars(r)["project_id"]
	if projectUUID == "" {
		http.Error(w, "project ID missing", 400)
		return nil, false
	}

	project = &db.Project{}
	err := db.DB.SelectOne(project, `SELECT * FROM projects WHERE uuid = $1`, projectUUID)
	switch {
	case err == sql.ErrNoRows:
		return nil, true
	case err == nil && domain.ID != project.DomainID:
		http.Error(w, "no such project", 404)
		return nil, false
	case ReturnError(w, err):
		return nil, false
	default:
		return project, true
	}
}
<|endoftext|>