<commit_before>package deque

// minCapacity is the smallest capacity that deque may have.
// Must be power of 2 for bitwise modulus: x % n == x & (n - 1).
const minCapacity = 16

// Deque represents a single instance of the deque data structure.
type Deque struct {
	buf   []interface{}
	head  int
	tail  int
	count int
}

// Len returns the number of elements currently stored in the queue.
func (q *Deque) Len() int {
	return q.count
}

// PushBack appends an element to the back of the queue.  Implements FIFO when
// elements are removed with PopFront(), and LIFO when elements are removed
// with PopBack().
func (q *Deque) PushBack(elem interface{}) {
	q.growIfFull()

	q.buf[q.tail] = elem
	// Calculate new tail position.
	q.tail = q.next(q.tail)
	q.count++
}

// PushFront prepends an element to the front of the queue.
func (q *Deque) PushFront(elem interface{}) {
	q.growIfFull()

	// Calculate new head position.
	q.head = q.prev(q.head)
	q.buf[q.head] = elem
	q.count++
}

// PopFront removes and returns the element from the front of the queue.
// Implements FIFO when used with PushBack().  If the queue is empty, the call
// panics.
func (q *Deque) PopFront() interface{} {
	if q.count <= 0 {
		panic("deque: PopFront() called on empty queue")
	}
	ret := q.buf[q.head]
	q.buf[q.head] = nil
	// Calculate new head position.
	q.head = q.next(q.head)
	q.count--

	q.shrinkIfExcess()
	return ret
}

// PopBack removes and returns the element from the back of the queue.
// Implements LIFO when used with PushBack().  If the queue is empty, the call
// panics.
func (q *Deque) PopBack() interface{} {
	if q.count <= 0 {
		panic("deque: PopBack() called on empty queue")
	}

	// Calculate new tail position
	q.tail = q.prev(q.tail)

	// Remove value at tail.
	ret := q.buf[q.tail]
	q.buf[q.tail] = nil
	q.count--

	q.shrinkIfExcess()
	return ret
}

// Front returns the element at the front of the queue.  This is the element
// that would be returned by PopFront().  This call panics if the queue is
// empty.
func (q *Deque) Front() interface{} {
	if q.count <= 0 {
		panic("deque: Front() called when empty")
	}
	return q.buf[q.head]
}

// Back returns the element at the back of the queue.  This is the element
// that would be returned by PopBack().  This call panics if the queue is
// empty.
func (q *Deque) Back() interface{} {
	if q.count <= 0 {
		panic("deque: Back() called when empty")
	}
	return q.buf[q.prev(q.tail)]
}

// At returns the element at index i in the queue without removing the element
// from the queue.  This method accepts only non-negative index values.  At(0)
// refers to the first element and is the same as Front().  At(Len()-1) refers
// to the last element and is the same as Back().  If the index is invalid, the
// call panics.
//
// The purpose of At is to allow Deque to serve as a more general purpose
// circular buffer, where items are only added to and removed from the the ends
// of the deque, but may be read from any place within the deque.  Consider the
// case of a fixed-size circular log buffer: A new entry is pushed onto one end
// and when full the oldest is popped from the other end.  All the log entries
// in the buffer must be readable without altering the buffer contents.
func (q *Deque) At(i int) interface{} {
	if i < 0 || i >= q.count {
		panic("deque: At() called with index out of range")
	}
	// bitwise modulus
	return q.buf[(q.head+i)&(len(q.buf)-1)]
}

// Clear removes all elements from the queue, but retains the current capacity.
// This is useful when repeatedly reusing the queue at high frequency to avoid
// GC during reuse.  The queue will not be resized smaller as long as items are
// only added.  Only when items are removed is the queue subject to getting
// resized smaller.
func (q *Deque) Clear() {
	// bitwise modulus
	modBits := len(q.buf) - 1
	for h := q.head; h != q.tail; h = (h + 1) & modBits {
		q.buf[h] = nil
	}
	q.head = 0
	q.tail = 0
	q.count = 0
}

// Rotate rotates the deque n steps front-to-back.  If n is negative, rotates
// back-to-front.  Having Deque provide Rotate() avoids resizing, that could
// not be may happen if implementing rotation using Pop and Push methods.
func (q *Deque) Rotate(n int) {
	if q.count <= 1 {
		return
	}
	// Rotating a multiple of q.count is same as no rotation.
	n %= q.count
	if n == 0 {
		return
	}

	modBits := len(q.buf) - 1
	// If no empty space in buffer, only move head and tail indexes.
	if q.head == q.tail {
		// Calculate new head and tail using bitwise modulus.
		q.head = (q.head + n) & modBits
		q.tail = (q.tail + n) & modBits
		return
	}

	if n < 0 {
		// Rotate back to front.
		for ; n < 0; n++ {
			// Calculate new head and tail using bitwise modulus.
			q.head = (q.head - 1) & modBits
			q.tail = (q.tail - 1) & modBits
			// Put tail value at head and remove value at tail.
			q.buf[q.head] = q.buf[q.tail]
			q.buf[q.tail] = nil
		}
		return
	}

	// Rotate front to back.
	for ; n > 0; n-- {
		// Put head value at tail and remove value at head.
		q.buf[q.tail] = q.buf[q.head]
		q.buf[q.head] = nil
		// Calculate new head and tail using bitwise modulus.
		q.head = (q.head + 1) & modBits
		q.tail = (q.tail + 1) & modBits
	}
}

// prev returns the previous buffer position wrapping around buffer.
func (q *Deque) prev(i int) int {
	return (i - 1) & (len(q.buf) - 1) // bitwise modulus
}

// next returns the next buffer position wrapping around buffer.
func (q *Deque) next(i int) int {
	return (i + 1) & (len(q.buf) - 1) // bitwise modulus
}

// growIfFull resizes up if the buffer is full.
func (q *Deque) growIfFull() {
	if len(q.buf) == 0 {
		q.buf = make([]interface{}, minCapacity)
		return
	}
	if q.count == len(q.buf) {
		q.resize()
	}
}

// shrinkIfExcess resize down if the buffer 1/4 full.
func (q *Deque) shrinkIfExcess() {
	if len(q.buf) > minCapacity && (q.count<<2) == len(q.buf) {
		q.resize()
	}
}

// resize resizes the deque to fit exactly twice its current contents.
// This results in shrinking if the queue is less than half-full, or growing
// the queue when it is full.
func (q *Deque) resize() {
	newBuf := make([]interface{}, q.count<<1)
	if q.tail > q.head {
		copy(newBuf, q.buf[q.head:q.tail])
	} else {
		n := copy(newBuf, q.buf[q.head:])
		copy(newBuf[n:], q.buf[:q.tail])
	}

	q.head = 0
	q.tail = q.count
	q.buf = newBuf
}
<commit_msg>fix typo in comment<commit_after>package deque

// minCapacity is the smallest capacity that deque may have.
// Must be power of 2 for bitwise modulus: x % n == x & (n - 1).
const minCapacity = 16

// Deque represents a single instance of the deque data structure.
type Deque struct {
	buf   []interface{}
	head  int
	tail  int
	count int
}

// Len returns the number of elements currently stored in the queue.
func (q *Deque) Len() int {
	return q.count
}

// PushBack appends an element to the back of the queue.  Implements FIFO when
// elements are removed with PopFront(), and LIFO when elements are removed
// with PopBack().
func (q *Deque) PushBack(elem interface{}) {
	q.growIfFull()

	q.buf[q.tail] = elem
	// Calculate new tail position.
	q.tail = q.next(q.tail)
	q.count++
}

// PushFront prepends an element to the front of the queue.
func (q *Deque) PushFront(elem interface{}) {
	q.growIfFull()

	// Calculate new head position.
	q.head = q.prev(q.head)
	q.buf[q.head] = elem
	q.count++
}

// PopFront removes and returns the element from the front of the queue.
// Implements FIFO when used with PushBack().  If the queue is empty, the call
// panics.
func (q *Deque) PopFront() interface{} {
	if q.count <= 0 {
		panic("deque: PopFront() called on empty queue")
	}
	ret := q.buf[q.head]
	q.buf[q.head] = nil
	// Calculate new head position.
	q.head = q.next(q.head)
	q.count--

	q.shrinkIfExcess()
	return ret
}

// PopBack removes and returns the element from the back of the queue.
// Implements LIFO when used with PushBack().  If the queue is empty, the call
// panics.
func (q *Deque) PopBack() interface{} {
	if q.count <= 0 {
		panic("deque: PopBack() called on empty queue")
	}

	// Calculate new tail position
	q.tail = q.prev(q.tail)

	// Remove value at tail.
	ret := q.buf[q.tail]
	q.buf[q.tail] = nil
	q.count--

	q.shrinkIfExcess()
	return ret
}

// Front returns the element at the front of the queue.  This is the element
// that would be returned by PopFront().  This call panics if the queue is
// empty.
func (q *Deque) Front() interface{} {
	if q.count <= 0 {
		panic("deque: Front() called when empty")
	}
	return q.buf[q.head]
}

// Back returns the element at the back of the queue.  This is the element
// that would be returned by PopBack().  This call panics if the queue is
// empty.
func (q *Deque) Back() interface{} {
	if q.count <= 0 {
		panic("deque: Back() called when empty")
	}
	return q.buf[q.prev(q.tail)]
}

// At returns the element at index i in the queue without removing the element
// from the queue.  This method accepts only non-negative index values.  At(0)
// refers to the first element and is the same as Front().  At(Len()-1) refers
// to the last element and is the same as Back().  If the index is invalid, the
// call panics.
//
// The purpose of At is to allow Deque to serve as a more general purpose
// circular buffer, where items are only added to and removed from the the ends
// of the deque, but may be read from any place within the deque.  Consider the
// case of a fixed-size circular log buffer: A new entry is pushed onto one end
// and when full the oldest is popped from the other end.  All the log entries
// in the buffer must be readable without altering the buffer contents.
func (q *Deque) At(i int) interface{} {
	if i < 0 || i >= q.count {
		panic("deque: At() called with index out of range")
	}
	// bitwise modulus
	return q.buf[(q.head+i)&(len(q.buf)-1)]
}

// Clear removes all elements from the queue, but retains the current capacity.
// This is useful when repeatedly reusing the queue at high frequency to avoid
// GC during reuse.  The queue will not be resized smaller as long as items are
// only added.  Only when items are removed is the queue subject to getting
// resized smaller.
func (q *Deque) Clear() {
	// bitwise modulus
	modBits := len(q.buf) - 1
	for h := q.head; h != q.tail; h = (h + 1) & modBits {
		q.buf[h] = nil
	}
	q.head = 0
	q.tail = 0
	q.count = 0
}

// Rotate rotates the deque n steps front-to-back.  If n is negative, rotates
// back-to-front.  Having Deque provide Rotate() avoids resizing that could
// happen if implementing rotation using only Pop and Push methods.
func (q *Deque) Rotate(n int) {
	if q.count <= 1 {
		return
	}
	// Rotating a multiple of q.count is same as no rotation.
	n %= q.count
	if n == 0 {
		return
	}

	modBits := len(q.buf) - 1
	// If no empty space in buffer, only move head and tail indexes.
	if q.head == q.tail {
		// Calculate new head and tail using bitwise modulus.
		q.head = (q.head + n) & modBits
		q.tail = (q.tail + n) & modBits
		return
	}

	if n < 0 {
		// Rotate back to front.
		for ; n < 0; n++ {
			// Calculate new head and tail using bitwise modulus.
			q.head = (q.head - 1) & modBits
			q.tail = (q.tail - 1) & modBits
			// Put tail value at head and remove value at tail.
			q.buf[q.head] = q.buf[q.tail]
			q.buf[q.tail] = nil
		}
		return
	}

	// Rotate front to back.
	for ; n > 0; n-- {
		// Put head value at tail and remove value at head.
		q.buf[q.tail] = q.buf[q.head]
		q.buf[q.head] = nil
		// Calculate new head and tail using bitwise modulus.
		q.head = (q.head + 1) & modBits
		q.tail = (q.tail + 1) & modBits
	}
}

// prev returns the previous buffer position wrapping around buffer.
func (q *Deque) prev(i int) int {
	return (i - 1) & (len(q.buf) - 1) // bitwise modulus
}

// next returns the next buffer position wrapping around buffer.
func (q *Deque) next(i int) int {
	return (i + 1) & (len(q.buf) - 1) // bitwise modulus
}

// growIfFull resizes up if the buffer is full.
func (q *Deque) growIfFull() {
	if len(q.buf) == 0 {
		q.buf = make([]interface{}, minCapacity)
		return
	}
	if q.count == len(q.buf) {
		q.resize()
	}
}

// shrinkIfExcess resize down if the buffer 1/4 full.
func (q *Deque) shrinkIfExcess() {
	if len(q.buf) > minCapacity && (q.count<<2) == len(q.buf) {
		q.resize()
	}
}

// resize resizes the deque to fit exactly twice its current contents.
// This results in shrinking if the queue is less than half-full, or growing
// the queue when it is full.
func (q *Deque) resize() {
	newBuf := make([]interface{}, q.count<<1)
	if q.tail > q.head {
		copy(newBuf, q.buf[q.head:q.tail])
	} else {
		n := copy(newBuf, q.buf[q.head:])
		copy(newBuf[n:], q.buf[:q.tail])
	}

	q.head = 0
	q.tail = q.count
	q.buf = newBuf
}
<|endoftext|>


<commit_before>package fwk

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"

	"github.com/go-hep/fwk/utils/tarjan"
)

type node struct {
	in  map[string]reflect.Type
	out map[string]reflect.Type
}

func newNode() *node {
	return &node{
		in:  make(map[string]reflect.Type),
		out: make(map[string]reflect.Type),
	}
}

type dflowsvc struct {
	SvcBase
	nodes map[string]*node
	edges map[string]reflect.Type
}

func (svc *dflowsvc) Configure(ctx Context) error {
	return nil
}

func (svc *dflowsvc) StartSvc(ctx Context) error {
	var err error

	// sort node-names for reproducibility
	nodenames := make([]string, 0, len(svc.nodes))
	for n := range svc.nodes {
		nodenames = append(nodenames, n)
	}
	sort.Strings(nodenames)

	// - make sure all input keys of components are available
	//   as output keys of a task
	// - also detect whether a key is labeled as an out-port
	//   by 2 different components
	out := make(map[string]string) // outport-name -> producer-name
	for _, tsk := range nodenames {
		node := svc.nodes[tsk]
		for k := range node.out {
			n, dup := out[k]
			if dup {
				return Errorf("%s: component [%s] already declared port [%s] as its output (current=%s)",
					svc.Name(), n, k, tsk,
				)
			}
			out[k] = tsk
		}
	}

	for _, tsk := range nodenames {
		node := svc.nodes[tsk]
		for k := range node.in {
			_, ok := out[k]
			if !ok {
				return Errorf("%s: component [%s] declared port [%s] as input but NO KNOWN producer",
					svc.Name(), tsk, k,
				)
			}
		}
	}

	// detect cycles.
	graph := make(map[interface{}][]interface{})
	for _, n := range nodenames {
		node := svc.nodes[n]
		graph[n] = []interface{}{}
		for in := range node.in {
			for _, o := range nodenames {
				if o == n {
					continue
				}
				onode := svc.nodes[o]
				connected := false
				for out := range onode.out {
					if in == out {
						connected = true
						break
					}
				}
				if connected {
					graph[n] = append(graph[n], o)
				}
			}
		}
	}

	cycles := tarjan.Connections(graph)
	if len(cycles) > 0 {
		msg := ctx.Msg()
		ncycles := 0
		for _, cycle := range cycles {
			if len(cycle) > 1 {
				ncycles += 1
				msg.Errorf("cycle detected: %v\n", cycle)
			}
		}
		s := ""
		if ncycles > 1 {
			s = "s"
		}
		if ncycles > 0 {
			return Errorf("%s: cycle%s detected: %d", svc.Name(), s, ncycles)
		}
	}

	return err
}

func (svc *dflowsvc) StopSvc(ctx Context) error {
	return nil
}

func (svc *dflowsvc) keys() []string {
	keys := make([]string, 0, len(svc.edges))
	for k := range svc.edges {
		keys = append(keys, k)
	}
	return keys
}

func (svc *dflowsvc) addInNode(tsk string, name string, t reflect.Type) error {
	node, ok := svc.nodes[tsk]
	if !ok {
		node = newNode()
		svc.nodes[tsk] = node
	}
	_, ok = node.in[name]
	if ok {
		return Errorf(
			"fwk.DeclInPort: component [%s] already declared in-port with name [%s]",
			tsk,
			name,
		)
	}

	node.in[name] = t
	edgetyp, dup := svc.edges[name]
	if dup {
		// make sure types match
		if edgetyp != t {
			type elem_t struct {
				port string // in/out
				task string // task which defined the port
				typ  reflect.Type
			}
			cont := []elem_t{}
			for tskname, node := range svc.nodes {
				for k, in := range node.in {
					if k != name {
						continue
					}
					cont = append(cont,
						elem_t{
							port: "in ",
							task: tskname,
							typ:  in,
						},
					)
				}
				for k, out := range node.out {
					if k != name {
						continue
					}
					cont = append(cont,
						elem_t{
							port: "out",
							task: tskname,
							typ:  out,
						},
					)
				}
			}
			var o bytes.Buffer
			fmt.Fprintf(&o, "fwk.DeclInPort: detected type inconsistency for port [%s]:\n", name)
			for _, c := range cont {
				fmt.Fprintf(&o, " component=%q port=%s type=%v\n", c.task, c.port, c.typ)
			}
			return fmt.Errorf(string(o.Bytes()))
		}
	}

	svc.edges[name] = t
	return nil
}

func (svc *dflowsvc) addOutNode(tsk string, name string, t reflect.Type) error {
	node, ok := svc.nodes[tsk]
	if !ok {
		node = newNode()
		svc.nodes[tsk] = node
	}
	_, ok = node.out[name]
	if ok {
		return Errorf(
			"fwk.DeclOutPort: component [%s] already declared out-port with name [%s]",
			tsk,
			name,
		)
	}

	node.out[name] = t

	edgetyp, dup := svc.edges[name]
	if dup {
		// edge already exists
		// loop over nodes, find out who already defined that edge
		for duptsk, dupnode := range svc.nodes {
			if duptsk == tsk {
				continue
			}
			for out := range dupnode.out {
				if out == name {
					return Errorf(
						"fwk.DeclOutPort: component [%s] already declared out-port with name [%s (type=%v)].\nfwk.DeclOutPort: component [%s] is trying to add a duplicate out-port [%s (type=%v)]",
						duptsk,
						name,
						edgetyp,
						tsk,
						name,
						t,
					)
				}
			}
		}
	}
	svc.edges[name] = t
	return nil
}

func init() {
	Register(reflect.TypeOf(dflowsvc{}),
		func(t, name string, mgr App) (Component, error) {
			svc := &dflowsvc{
				SvcBase: NewSvc(t, name, mgr),
				nodes:   make(map[string]*node),
				edges:   make(map[string]reflect.Type),
			}
			return svc, nil
		},
	)
}

// EOF
<commit_msg>dflow: sort on nodenames for reproducibility<commit_after>package fwk

import (
	"bytes"
	"fmt"
	"reflect"
	"sort"

	"github.com/go-hep/fwk/utils/tarjan"
)

type node struct {
	in  map[string]reflect.Type
	out map[string]reflect.Type
}

func newNode() *node {
	return &node{
		in:  make(map[string]reflect.Type),
		out: make(map[string]reflect.Type),
	}
}

type dflowsvc struct {
	SvcBase
	nodes map[string]*node
	edges map[string]reflect.Type
}

func (svc *dflowsvc) Configure(ctx Context) error {
	return nil
}

func (svc *dflowsvc) StartSvc(ctx Context) error {
	var err error

	// sort node-names for reproducibility
	nodenames := make([]string, 0, len(svc.nodes))
	for n := range svc.nodes {
		nodenames = append(nodenames, n)
	}
	sort.Strings(nodenames)

	// - make sure all input keys of components are available
	//   as output keys of a task
	// - also detect whether a key is labeled as an out-port
	//   by 2 different components
	out := make(map[string]string) // outport-name -> producer-name
	for _, tsk := range nodenames {
		node := svc.nodes[tsk]
		for k := range node.out {
			n, dup := out[k]
			if dup {
				return Errorf("%s: component [%s] already declared port [%s] as its output (current=%s)",
					svc.Name(), n, k, tsk,
				)
			}
			out[k] = tsk
		}
	}

	for _, tsk := range nodenames {
		node := svc.nodes[tsk]
		for k := range node.in {
			_, ok := out[k]
			if !ok {
				return Errorf("%s: component [%s] declared port [%s] as input but NO KNOWN producer",
					svc.Name(), tsk, k,
				)
			}
		}
	}

	// detect cycles.
	graph := make(map[interface{}][]interface{})
	for _, n := range nodenames {
		node := svc.nodes[n]
		graph[n] = []interface{}{}
		for in := range node.in {
			for _, o := range nodenames {
				if o == n {
					continue
				}
				onode := svc.nodes[o]
				connected := false
				for out := range onode.out {
					if in == out {
						connected = true
						break
					}
				}
				if connected {
					graph[n] = append(graph[n], o)
				}
			}
		}
	}

	cycles := tarjan.Connections(graph)
	if len(cycles) > 0 {
		msg := ctx.Msg()
		ncycles := 0
		for _, cycle := range cycles {
			if len(cycle) > 1 {
				ncycles += 1
				msg.Errorf("cycle detected: %v\n", cycle)
			}
		}
		s := ""
		if ncycles > 1 {
			s = "s"
		}
		if ncycles > 0 {
			return Errorf("%s: cycle%s detected: %d", svc.Name(), s, ncycles)
		}
	}

	return err
}

func (svc *dflowsvc) StopSvc(ctx Context) error {
	return nil
}

func (svc *dflowsvc) keys() []string {
	keys := make([]string, 0, len(svc.edges))
	for k := range svc.edges {
		keys = append(keys, k)
	}
	return keys
}

func (svc *dflowsvc) addInNode(tsk string, name string, t reflect.Type) error {
	node, ok := svc.nodes[tsk]
	if !ok {
		node = newNode()
		svc.nodes[tsk] = node
	}
	_, ok = node.in[name]
	if ok {
		return Errorf(
			"fwk.DeclInPort: component [%s] already declared in-port with name [%s]",
			tsk,
			name,
		)
	}

	node.in[name] = t
	edgetyp, dup := svc.edges[name]
	if dup {
		// make sure types match
		if edgetyp != t {
			type elem_t struct {
				port string // in/out
				task string // task which defined the port
				typ  reflect.Type
			}
			cont := []elem_t{}
			nodenames := make([]string, 0, len(svc.nodes))
			for tskname := range svc.nodes {
				nodenames = append(nodenames, tskname)
			}
			sort.Strings(nodenames)
			for _, tskname := range nodenames {
				node := svc.nodes[tskname]
				for k, in := range node.in {
					if k != name {
						continue
					}
					cont = append(cont,
						elem_t{
							port: "in ",
							task: tskname,
							typ:  in,
						},
					)
				}
				for k, out := range node.out {
					if k != name {
						continue
					}
					cont = append(cont,
						elem_t{
							port: "out",
							task: tskname,
							typ:  out,
						},
					)
				}
			}
			var o bytes.Buffer
			fmt.Fprintf(&o, "fwk.DeclInPort: detected type inconsistency for port [%s]:\n", name)
			for _, c := range cont {
				fmt.Fprintf(&o, " component=%q port=%s type=%v\n", c.task, c.port, c.typ)
			}
			return fmt.Errorf(string(o.Bytes()))
		}
	}

	svc.edges[name] = t
	return nil
}

func (svc *dflowsvc) addOutNode(tsk string, name string, t reflect.Type) error {
	node, ok := svc.nodes[tsk]
	if !ok {
		node = newNode()
		svc.nodes[tsk] = node
	}
	_, ok = node.out[name]
	if ok {
		return Errorf(
			"fwk.DeclOutPort: component [%s] already declared out-port with name [%s]",
			tsk,
			name,
		)
	}

	node.out[name] = t

	edgetyp, dup := svc.edges[name]
	if dup {
		// edge already exists
		// loop over nodes, find out who already defined that edge
		nodenames := make([]string, 0, len(svc.nodes))
		for tskname := range svc.nodes {
			nodenames = append(nodenames, tskname)
		}
		sort.Strings(nodenames)
		for _, duptsk := range nodenames {
			dupnode := svc.nodes[duptsk]
			if duptsk == tsk {
				continue
			}
			for out := range dupnode.out {
				if out == name {
					return Errorf(
						"fwk.DeclOutPort: component [%s] already declared out-port with name [%s (type=%v)].\nfwk.DeclOutPort: component [%s] is trying to add a duplicate out-port [%s (type=%v)]",
						duptsk,
						name,
						edgetyp,
						tsk,
						name,
						t,
					)
				}
			}
		}
	}
	svc.edges[name] = t
	return nil
}

func init() {
	Register(reflect.TypeOf(dflowsvc{}),
		func(t, name string, mgr App) (Component, error) {
			svc := &dflowsvc{
				SvcBase: NewSvc(t, name, mgr),
				nodes:   make(map[string]*node),
				edges:   make(map[string]reflect.Type),
			}
			return svc, nil
		},
	)
}

// EOF
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func init() {
	resource.AddTestSweepers("aws_cloudwatch_log_resource_policy", &resource.Sweeper{
		Name: "aws_cloudwatch_log_resource_policy",
		F:    testSweepCloudWatchLogResourcePolicys,
	})
}

func testSweepCloudWatchLogResourcePolicys(region string) error {
	client, err := sharedClientForRegion(region)
	if err != nil {
		return fmt.Errorf("error getting client: %s", err)
	}
	conn := client.(*AWSClient).cloudwatchlogsconn

	input := &cloudwatchlogs.DescribeResourcePoliciesInput{}

	for {
		output, err := conn.DescribeResourcePolicies(input)
		if testSweepSkipSweepError(err) {
			log.Printf("[WARN] Skipping CloudWatchLog Resource Policy sweep for %s: %s", region, err)
			return nil
		}

		if err != nil {
			return fmt.Errorf("error describing CloudWatchLog Resource Policy: %s", err)
		}

		for _, resourcePolicy := range output.ResourcePolicies {
			deleteInput := &cloudwatchlogs.DeleteResourcePolicyInput{
				PolicyName: resourcePolicy.PolicyName,
			}
			if _, err := conn.DeleteResourcePolicy(deleteInput); err != nil {
				return fmt.Errorf("error deleting CloudWatch log resource policy (%s): %s", aws.StringValue(resourcePolicy.PolicyName), err)
			}
		}

		if output.NextToken == nil {
			break
		}

		input.NextToken = output.NextToken
	}

	return nil
}

func TestAccAWSCloudWatchLogResourcePolicy_Basic(t *testing.T) {
	name := acctest.RandString(5)
	var resourcePolicy cloudwatchlogs.ResourcePolicy
	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckCloudWatchLogResourcePolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic1(name),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckCloudWatchLogResourcePolicy("aws_cloudwatch_log_resource_policy.test", &resourcePolicy),
					resource.TestCheckResourceAttr("aws_cloudwatch_log_resource_policy.test", "policy_name", name),
					resource.TestCheckResourceAttr("aws_cloudwatch_log_resource_policy.test", "policy_document", "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"route53.amazonaws.com\"},\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:*:*:log-group:/aws/route53/*\"}]}"),
				),
			},
			{
				Config: testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic2(name),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckCloudWatchLogResourcePolicy("aws_cloudwatch_log_resource_policy.test", &resourcePolicy),
					resource.TestCheckResourceAttr("aws_cloudwatch_log_resource_policy.test", "policy_name", name),
					resource.TestCheckResourceAttr("aws_cloudwatch_log_resource_policy.test", "policy_document", "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"route53.amazonaws.com\"},\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:*:*:log-group:/aws/route53/example.com\"}]}"),
				),
			},
		},
	})
}

func TestAccAWSCloudWatchLogResourcePolicy_Import(t *testing.T) {
	resourceName := "aws_cloudwatch_log_resource_policy.test"

	name := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckCloudWatchLogResourcePolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic1(name),
			},

			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccCheckCloudWatchLogResourcePolicy(pr string, resourcePolicy *cloudwatchlogs.ResourcePolicy) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).cloudwatchlogsconn
		rs, ok := s.RootModule().Resources[pr]
		if !ok {
			return fmt.Errorf("Not found: %s", pr)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		policy, exists, err := lookupCloudWatchLogResourcePolicy(conn, rs.Primary.ID, nil)
		if err != nil {
			return err
		}
		if !exists {
			return fmt.Errorf("Resource policy does not exist: %q", rs.Primary.ID)
		}

		*resourcePolicy = *policy

		return nil
	}
}

func testAccCheckCloudWatchLogResourcePolicyDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).cloudwatchlogsconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_cloudwatch_log_resource_policy" {
			continue
		}

		_, exists, err := lookupCloudWatchLogResourcePolicy(conn, rs.Primary.ID, nil)
		if err != nil {
			return nil
		}

		if exists {
			return fmt.Errorf("Resource policy exists: %q", rs.Primary.ID)
		}
	}

	return nil
}

func testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic1(name string) string {
	return fmt.Sprintf(`
data "aws_iam_policy_document" "test" {
  statement {
    actions = [
      "logs:CreateLogStream",
      "logs:PutLogEvents",
    ]

    resources = ["arn:aws:logs:*:*:log-group:/aws/route53/*"]

    principals {
      identifiers = ["route53.amazonaws.com"]
      type        = "Service"
    }
  }
}

resource "aws_cloudwatch_log_resource_policy" "test" {
  policy_name     = "%s"
  policy_document = "${data.aws_iam_policy_document.test.json}"
}
`, name)
}

func testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic2(name string) string {
	return fmt.Sprintf(`
data "aws_iam_policy_document" "test" {
  statement {
    actions = [
      "logs:CreateLogStream",
      "logs:PutLogEvents",
    ]

    resources = ["arn:aws:logs:*:*:log-group:/aws/route53/example.com"]

    principals {
      identifiers = ["route53.amazonaws.com"]
      type        = "Service"
    }
  }
}

resource "aws_cloudwatch_log_resource_policy" "test" {
  policy_name     = "%s"
  policy_document = "${data.aws_iam_policy_document.test.json}"
}
`, name)
}
<commit_msg>tests/resource/aws_cloudwatch_log_resource_policy: Minor adjustments to sweeper to log deletions and fix typo<commit_after>package aws

import (
	"fmt"
	"log"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"

	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func init() {
	resource.AddTestSweepers("aws_cloudwatch_log_resource_policy", &resource.Sweeper{
		Name: "aws_cloudwatch_log_resource_policy",
		F:    testSweepCloudWatchLogResourcePolicies,
	})
}

func testSweepCloudWatchLogResourcePolicies(region string) error {
	client, err := sharedClientForRegion(region)
	if err != nil {
		return fmt.Errorf("error getting client: %s", err)
	}
	conn := client.(*AWSClient).cloudwatchlogsconn

	input := &cloudwatchlogs.DescribeResourcePoliciesInput{}

	for {
		output, err := conn.DescribeResourcePolicies(input)
		if testSweepSkipSweepError(err) {
			log.Printf("[WARN] Skipping CloudWatchLog Resource Policy sweep for %s: %s", region, err)
			return nil
		}

		if err != nil {
			return fmt.Errorf("error describing CloudWatchLog Resource Policy: %s", err)
		}

		for _, resourcePolicy := range output.ResourcePolicies {
			policyName := aws.StringValue(resourcePolicy.PolicyName)
			deleteInput := &cloudwatchlogs.DeleteResourcePolicyInput{
				PolicyName: resourcePolicy.PolicyName,
			}

			log.Printf("[INFO] Deleting CloudWatch Log Resource Policy: %s", policyName)

			if _, err := conn.DeleteResourcePolicy(deleteInput); err != nil {
				return fmt.Errorf("error deleting CloudWatch log resource policy (%s): %s", policyName, err)
			}
		}

		if aws.StringValue(output.NextToken) == "" {
			break
		}

		input.NextToken = output.NextToken
	}

	return nil
}

func TestAccAWSCloudWatchLogResourcePolicy_Basic(t *testing.T) {
	name := acctest.RandString(5)
	var resourcePolicy cloudwatchlogs.ResourcePolicy
	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckCloudWatchLogResourcePolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic1(name),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckCloudWatchLogResourcePolicy("aws_cloudwatch_log_resource_policy.test", &resourcePolicy),
					resource.TestCheckResourceAttr("aws_cloudwatch_log_resource_policy.test", "policy_name", name),
					resource.TestCheckResourceAttr("aws_cloudwatch_log_resource_policy.test", "policy_document", "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"route53.amazonaws.com\"},\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:*:*:log-group:/aws/route53/*\"}]}"),
				),
			},
			{
				Config: testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic2(name),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckCloudWatchLogResourcePolicy("aws_cloudwatch_log_resource_policy.test", &resourcePolicy),
					resource.TestCheckResourceAttr("aws_cloudwatch_log_resource_policy.test", "policy_name", name),
					resource.TestCheckResourceAttr("aws_cloudwatch_log_resource_policy.test", "policy_document", "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Sid\":\"\",\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"route53.amazonaws.com\"},\"Action\":[\"logs:PutLogEvents\",\"logs:CreateLogStream\"],\"Resource\":\"arn:aws:logs:*:*:log-group:/aws/route53/example.com\"}]}"),
				),
			},
		},
	})
}

func TestAccAWSCloudWatchLogResourcePolicy_Import(t *testing.T) {
	resourceName := "aws_cloudwatch_log_resource_policy.test"

	name := acctest.RandString(5)

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckCloudWatchLogResourcePolicyDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic1(name),
			},

			{
				ResourceName:      resourceName,
				ImportState:       true,
				ImportStateVerify: true,
			},
		},
	})
}

func testAccCheckCloudWatchLogResourcePolicy(pr string, resourcePolicy *cloudwatchlogs.ResourcePolicy) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		conn := testAccProvider.Meta().(*AWSClient).cloudwatchlogsconn
		rs, ok := s.RootModule().Resources[pr]
		if !ok {
			return fmt.Errorf("Not found: %s", pr)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		policy, exists, err := lookupCloudWatchLogResourcePolicy(conn, rs.Primary.ID, nil)
		if err != nil {
			return err
		}
		if !exists {
			return fmt.Errorf("Resource policy does not exist: %q", rs.Primary.ID)
		}

		*resourcePolicy = *policy

		return nil
	}
}

func testAccCheckCloudWatchLogResourcePolicyDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).cloudwatchlogsconn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_cloudwatch_log_resource_policy" {
			continue
		}

		_, exists, err := lookupCloudWatchLogResourcePolicy(conn, rs.Primary.ID, nil)
		if err != nil {
			return nil
		}

		if exists {
			return fmt.Errorf("Resource policy exists: %q", rs.Primary.ID)
		}
	}

	return nil
}

func testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic1(name string) string {
	return fmt.Sprintf(`
data "aws_iam_policy_document" "test" {
  statement {
    actions = [
      "logs:CreateLogStream",
      "logs:PutLogEvents",
    ]

    resources = ["arn:aws:logs:*:*:log-group:/aws/route53/*"]

    principals {
      identifiers = ["route53.amazonaws.com"]
      type        = "Service"
    }
  }
}

resource "aws_cloudwatch_log_resource_policy" "test" {
  policy_name     = "%s"
  policy_document = "${data.aws_iam_policy_document.test.json}"
}
`, name)
}

func testAccCheckAWSCloudWatchLogResourcePolicyResourceConfigBasic2(name string) string {
	return fmt.Sprintf(`
data "aws_iam_policy_document" "test" {
  statement {
    actions = [
      "logs:CreateLogStream",
      "logs:PutLogEvents",
    ]

    resources = ["arn:aws:logs:*:*:log-group:/aws/route53/example.com"]

    principals {
      identifiers = ["route53.amazonaws.com"]
      type        = "Service"
    }
  }
}

resource "aws_cloudwatch_log_resource_policy" "test" {
  policy_name     = "%s"
  policy_document = "${data.aws_iam_policy_document.test.json}"
}
`, name)
}
<|endoftext|>


<commit_before>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"gopkg.in/yaml.v2"
)

// BasicAuth contains basic HTTP authentication credentials.
type BasicAuth struct {
	Username string `yaml:"username"`
	Password Secret `yaml:"password"`

	// Catches all undefined fields and must be empty after parsing.
	XXX map[string]interface{} `yaml:",inline"`
}

// URL is a custom URL type that allows validation at configuration load time.
type URL struct {
	*url.URL
}

// UnmarshalYAML implements the yaml.Unmarshaler interface for URLs.
func (u *URL) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var s string
	if err := unmarshal(&s); err != nil {
		return err
	}

	urlp, err := url.Parse(s)
	if err != nil {
		return err
	}
	u.URL = urlp
	return nil
}

// MarshalYAML implements the yaml.Marshaler interface for URLs.
func (u URL) MarshalYAML() (interface{}, error) {
	if u.URL != nil {
		return u.String(), nil
	}
	return nil, nil
}

// HTTPClientConfig configures an HTTP client.
type HTTPClientConfig struct {
	// The HTTP basic authentication credentials for the targets.
	BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"`
	// The bearer token for the targets.
	BearerToken Secret `yaml:"bearer_token,omitempty"`
	// The bearer token file for the targets.
	BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
	// HTTP proxy server to use to connect to the targets.
	ProxyURL URL `yaml:"proxy_url,omitempty"`
	// TLSConfig to use to connect to the targets.
	TLSConfig TLSConfig `yaml:"tls_config,omitempty"`

	// Catches all undefined fields and must be empty after parsing.
	XXX map[string]interface{} `yaml:",inline"`
}

// Validate validates the HTTPClientConfig to check only one of BearerToken,
// BasicAuth and BearerTokenFile is configured.
func (c *HTTPClientConfig) Validate() error {
	if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 {
		return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured")
	}
	if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) {
		return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured")
	}
	return nil
}

// UnmarshalYAML implements the yaml.Unmarshaler interface
func (c *HTTPClientConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain HTTPClientConfig
	err := unmarshal((*plain)(c))
	if err != nil {
		return err
	}
	err = c.Validate()
	if err != nil {
		return c.Validate()
	}
	return checkOverflow(c.XXX, "http_client_config")
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (a *BasicAuth) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain BasicAuth
	err := unmarshal((*plain)(a))
	if err != nil {
		return err
	}
	return checkOverflow(a.XXX, "basic_auth")
}

// NewHTTPClientFromConfig returns a new HTTP client configured for the
// given config.HTTPClientConfig.
func NewHTTPClientFromConfig(cfg *HTTPClientConfig) (*http.Client, error) {
	tlsConfig, err := NewTLSConfig(&cfg.TLSConfig)
	if err != nil {
		return nil, err
	}

	// It's the caller's job to handle timeouts
	var rt http.RoundTripper = &http.Transport{
		Proxy:             http.ProxyURL(cfg.ProxyURL.URL),
		DisableKeepAlives: true,
		TLSClientConfig:   tlsConfig,
	}

	// If a bearer token is provided, create a round tripper that will set the
	// Authorization header correctly on each request.
	bearerToken := cfg.BearerToken
	if len(bearerToken) == 0 && len(cfg.BearerTokenFile) > 0 {
		b, err := ioutil.ReadFile(cfg.BearerTokenFile)
		if err != nil {
			return nil, fmt.Errorf("unable to read bearer token file %s: %s", cfg.BearerTokenFile, err)
		}
		bearerToken = Secret(strings.TrimSpace(string(b)))
	}

	if len(bearerToken) > 0 {
		rt = NewBearerAuthRoundTripper(bearerToken, rt)
	}

	if cfg.BasicAuth != nil {
		rt = NewBasicAuthRoundTripper(cfg.BasicAuth.Username, Secret(cfg.BasicAuth.Password), rt)
	}

	// Return a new client with the configured round tripper.
	return &http.Client{Transport: rt}, nil
}

type bearerAuthRoundTripper struct {
	bearerToken Secret
	rt          http.RoundTripper
}

type basicAuthRoundTripper struct {
	username string
	password Secret
	rt       http.RoundTripper
}

// NewBasicAuthRoundTripper will apply a BASIC auth authorization header to a request unless it has
// already been set.
func NewBasicAuthRoundTripper(username string, password Secret, rt http.RoundTripper) http.RoundTripper {
	return &basicAuthRoundTripper{username, password, rt}
}

func (rt *bearerAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	if len(req.Header.Get("Authorization")) == 0 {
		req = cloneRequest(req)
		req.Header.Set("Authorization", "Bearer "+string(rt.bearerToken))
	}

	return rt.rt.RoundTrip(req)
}

// NewBearerAuthRoundTripper adds the provided bearer token to a request unless the authorization
// header has already been set.
func NewBearerAuthRoundTripper(bearer Secret, rt http.RoundTripper) http.RoundTripper {
	return &bearerAuthRoundTripper{bearer, rt}
}

func (rt *basicAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	if len(req.Header.Get("Authorization")) != 0 {
		return rt.RoundTrip(req)
	}
	req = cloneRequest(req)
	req.SetBasicAuth(rt.username, string(rt.password))
	return rt.rt.RoundTrip(req)
}

// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
	// Shallow copy of the struct.
	r2 := new(http.Request)
	*r2 = *r
	// Deep copy of the Header.
	r2.Header = make(http.Header)
	for k, s := range r.Header {
		r2.Header[k] = s
	}
	return r2
}

// NewTLSConfig creates a new tls.Config from the given config.TLSConfig.
func NewTLSConfig(cfg *TLSConfig) (*tls.Config, error) {
	tlsConfig := &tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify}

	// If a CA cert is provided then let's read it in so we can Validate the
	// scrape target's certificate properly.
	if len(cfg.CAFile) > 0 {
		caCertPool := x509.NewCertPool()
		// Load CA cert.
		caCert, err := ioutil.ReadFile(cfg.CAFile)
		if err != nil {
			return nil, fmt.Errorf("unable to use specified CA cert %s: %s", cfg.CAFile, err)
		}
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig.RootCAs = caCertPool
	}

	if len(cfg.ServerName) > 0 {
		tlsConfig.ServerName = cfg.ServerName
	}

	// If a client cert & key is provided then configure TLS config accordingly.
	if len(cfg.CertFile) > 0 && len(cfg.KeyFile) == 0 {
		return nil, fmt.Errorf("client cert file %q specified without client key file", cfg.CertFile)
	} else if len(cfg.KeyFile) > 0 && len(cfg.CertFile) == 0 {
		return nil, fmt.Errorf("client key file %q specified without client cert file", cfg.KeyFile)
	} else if len(cfg.CertFile) > 0 && len(cfg.KeyFile) > 0 {
		cert, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile)
		if err != nil {
			return nil, fmt.Errorf("unable to use specified client cert (%s) & key (%s): %s", cfg.CertFile, cfg.KeyFile, err)
		}
		tlsConfig.Certificates = []tls.Certificate{cert}
	}
	tlsConfig.BuildNameToCertificate()

	return tlsConfig, nil
}

// TLSConfig configures the options for TLS connections.
type TLSConfig struct {
	// The CA cert to use for the targets.
	CAFile string `yaml:"ca_file,omitempty"`
	// The client cert file for the targets.
	CertFile string `yaml:"cert_file,omitempty"`
	// The client key file for the targets.
	KeyFile string `yaml:"key_file,omitempty"`
	// Used to verify the hostname for the targets.
	ServerName string `yaml:"server_name,omitempty"`
	// Disable target certificate validation.
	InsecureSkipVerify bool `yaml:"insecure_skip_verify"`

	// Catches all undefined fields and must be empty after parsing.
	XXX map[string]interface{} `yaml:",inline"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *TLSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain TLSConfig
	if err := unmarshal((*plain)(c)); err != nil {
		return err
	}
	return checkOverflow(c.XXX, "TLS config")
}

func (c HTTPClientConfig) String() string {
	b, err := yaml.Marshal(c)
	if err != nil {
		return fmt.Sprintf("<error creating http client config string: %s>", err)
	}
	return string(b)
}
<commit_msg>nit: fix case in comment<commit_after>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"gopkg.in/yaml.v2"
)

// BasicAuth contains basic HTTP authentication credentials.
type BasicAuth struct {
	Username string `yaml:"username"`
	Password Secret `yaml:"password"`

	// Catches all undefined fields and must be empty after parsing.
	XXX map[string]interface{} `yaml:",inline"`
}

// URL is a custom URL type that allows validation at configuration load time.
type URL struct {
	*url.URL
}

// UnmarshalYAML implements the yaml.Unmarshaler interface for URLs.
func (u *URL) UnmarshalYAML(unmarshal func(interface{}) error) error {
	var s string
	if err := unmarshal(&s); err != nil {
		return err
	}

	urlp, err := url.Parse(s)
	if err != nil {
		return err
	}
	u.URL = urlp
	return nil
}

// MarshalYAML implements the yaml.Marshaler interface for URLs.
func (u URL) MarshalYAML() (interface{}, error) {
	if u.URL != nil {
		return u.String(), nil
	}
	return nil, nil
}

// HTTPClientConfig configures an HTTP client.
type HTTPClientConfig struct {
	// The HTTP basic authentication credentials for the targets.
	BasicAuth *BasicAuth `yaml:"basic_auth,omitempty"`
	// The bearer token for the targets.
	BearerToken Secret `yaml:"bearer_token,omitempty"`
	// The bearer token file for the targets.
	BearerTokenFile string `yaml:"bearer_token_file,omitempty"`
	// HTTP proxy server to use to connect to the targets.
	ProxyURL URL `yaml:"proxy_url,omitempty"`
	// TLSConfig to use to connect to the targets.
	TLSConfig TLSConfig `yaml:"tls_config,omitempty"`

	// Catches all undefined fields and must be empty after parsing.
	XXX map[string]interface{} `yaml:",inline"`
}

// Validate validates the HTTPClientConfig to check only one of BearerToken,
// BasicAuth and BearerTokenFile is configured.
func (c *HTTPClientConfig) Validate() error {
	if len(c.BearerToken) > 0 && len(c.BearerTokenFile) > 0 {
		return fmt.Errorf("at most one of bearer_token & bearer_token_file must be configured")
	}
	if c.BasicAuth != nil && (len(c.BearerToken) > 0 || len(c.BearerTokenFile) > 0) {
		return fmt.Errorf("at most one of basic_auth, bearer_token & bearer_token_file must be configured")
	}
	return nil
}

// UnmarshalYAML implements the yaml.Unmarshaler interface
func (c *HTTPClientConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain HTTPClientConfig
	err := unmarshal((*plain)(c))
	if err != nil {
		return err
	}
	err = c.Validate()
	if err != nil {
		return c.Validate()
	}
	return checkOverflow(c.XXX, "http_client_config")
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (a *BasicAuth) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain BasicAuth
	err := unmarshal((*plain)(a))
	if err != nil {
		return err
	}
	return checkOverflow(a.XXX, "basic_auth")
}

// NewHTTPClientFromConfig returns a new HTTP client configured for the
// given config.HTTPClientConfig.
func NewHTTPClientFromConfig(cfg *HTTPClientConfig) (*http.Client, error) {
	tlsConfig, err := NewTLSConfig(&cfg.TLSConfig)
	if err != nil {
		return nil, err
	}

	// It's the caller's job to handle timeouts
	var rt http.RoundTripper = &http.Transport{
		Proxy:             http.ProxyURL(cfg.ProxyURL.URL),
		DisableKeepAlives: true,
		TLSClientConfig:   tlsConfig,
	}

	// If a bearer token is provided, create a round tripper that will set the
	// Authorization header correctly on each request.
	bearerToken := cfg.BearerToken
	if len(bearerToken) == 0 && len(cfg.BearerTokenFile) > 0 {
		b, err := ioutil.ReadFile(cfg.BearerTokenFile)
		if err != nil {
			return nil, fmt.Errorf("unable to read bearer token file %s: %s", cfg.BearerTokenFile, err)
		}
		bearerToken = Secret(strings.TrimSpace(string(b)))
	}

	if len(bearerToken) > 0 {
		rt = NewBearerAuthRoundTripper(bearerToken, rt)
	}

	if cfg.BasicAuth != nil {
		rt = NewBasicAuthRoundTripper(cfg.BasicAuth.Username, Secret(cfg.BasicAuth.Password), rt)
	}

	// Return a new client with the configured round tripper.
	return &http.Client{Transport: rt}, nil
}

type bearerAuthRoundTripper struct {
	bearerToken Secret
	rt          http.RoundTripper
}

type basicAuthRoundTripper struct {
	username string
	password Secret
	rt       http.RoundTripper
}

// NewBasicAuthRoundTripper will apply a BASIC auth authorization header to a request unless it has
// already been set.
func NewBasicAuthRoundTripper(username string, password Secret, rt http.RoundTripper) http.RoundTripper {
	return &basicAuthRoundTripper{username, password, rt}
}

func (rt *bearerAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	if len(req.Header.Get("Authorization")) == 0 {
		req = cloneRequest(req)
		req.Header.Set("Authorization", "Bearer "+string(rt.bearerToken))
	}

	return rt.rt.RoundTrip(req)
}

// NewBearerAuthRoundTripper adds the provided bearer token to a request unless the authorization
// header has already been set.
func NewBearerAuthRoundTripper(bearer Secret, rt http.RoundTripper) http.RoundTripper {
	return &bearerAuthRoundTripper{bearer, rt}
}

func (rt *basicAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	if len(req.Header.Get("Authorization")) != 0 {
		return rt.RoundTrip(req)
	}
	req = cloneRequest(req)
	req.SetBasicAuth(rt.username, string(rt.password))
	return rt.rt.RoundTrip(req)
}

// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
	// Shallow copy of the struct.
	r2 := new(http.Request)
	*r2 = *r
	// Deep copy of the Header.
	r2.Header = make(http.Header)
	for k, s := range r.Header {
		r2.Header[k] = s
	}
	return r2
}

// NewTLSConfig creates a new tls.Config from the given config.TLSConfig.
func NewTLSConfig(cfg *TLSConfig) (*tls.Config, error) {
	tlsConfig := &tls.Config{InsecureSkipVerify: cfg.InsecureSkipVerify}

	// If a CA cert is provided then let's read it in so we can validate the
	// scrape target's certificate properly.
	if len(cfg.CAFile) > 0 {
		caCertPool := x509.NewCertPool()
		// Load CA cert.
		caCert, err := ioutil.ReadFile(cfg.CAFile)
		if err != nil {
			return nil, fmt.Errorf("unable to use specified CA cert %s: %s", cfg.CAFile, err)
		}
		caCertPool.AppendCertsFromPEM(caCert)
		tlsConfig.RootCAs = caCertPool
	}

	if len(cfg.ServerName) > 0 {
		tlsConfig.ServerName = cfg.ServerName
	}

	// If a client cert & key is provided then configure TLS config accordingly.
	if len(cfg.CertFile) > 0 && len(cfg.KeyFile) == 0 {
		return nil, fmt.Errorf("client cert file %q specified without client key file", cfg.CertFile)
	} else if len(cfg.KeyFile) > 0 && len(cfg.CertFile) == 0 {
		return nil, fmt.Errorf("client key file %q specified without client cert file", cfg.KeyFile)
	} else if len(cfg.CertFile) > 0 && len(cfg.KeyFile) > 0 {
		cert, err := tls.LoadX509KeyPair(cfg.CertFile, cfg.KeyFile)
		if err != nil {
			return nil, fmt.Errorf("unable to use specified client cert (%s) & key (%s): %s", cfg.CertFile, cfg.KeyFile, err)
		}
		tlsConfig.Certificates = []tls.Certificate{cert}
	}
	tlsConfig.BuildNameToCertificate()

	return tlsConfig, nil
}

// TLSConfig configures the options for TLS connections.
type TLSConfig struct {
	// The CA cert to use for the targets.
	CAFile string `yaml:"ca_file,omitempty"`
	// The client cert file for the targets.
	CertFile string `yaml:"cert_file,omitempty"`
	// The client key file for the targets.
	KeyFile string `yaml:"key_file,omitempty"`
	// Used to verify the hostname for the targets.
	ServerName string `yaml:"server_name,omitempty"`
	// Disable target certificate validation.
	InsecureSkipVerify bool `yaml:"insecure_skip_verify"`

	// Catches all undefined fields and must be empty after parsing.
	XXX map[string]interface{} `yaml:",inline"`
}

// UnmarshalYAML implements the yaml.Unmarshaler interface.
func (c *TLSConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
	type plain TLSConfig
	if err := unmarshal((*plain)(c)); err != nil {
		return err
	}
	return checkOverflow(c.XXX, "TLS config")
}

func (c HTTPClientConfig) String() string {
	b, err := yaml.Marshal(c)
	if err != nil {
		return fmt.Sprintf("<error creating http client config string: %s>", err)
	}
	return string(b)
}
<|endoftext|>


<commit_before>package haigo

import (
	"bytes"
	"encoding/json"
	"fmt"
	"text/template"

	"gopkg.in/mgo.v2"
	"gopkg.in/yaml.v2"
)

type param struct {
	Name string
	Type string
}

type Query struct {
	Name        string  `yaml:"name"`
	Description string  `yaml:"description,omitempty"`
	QueryString string  `yaml:"query"`
	params      []param // TODO
}

type Params map[string]interface{}

// Returns configured mgo Pipe.
func (h *Query) Pipe(col *mgo.Collection, params Params) (*mgo.Pipe, error) {

	q, err := h.Map(params)
	if err != nil {
		return nil, err
	}

	return col.Pipe(q), nil
}

// Returns configured mgo Query.
func (h *Query) Query(col *mgo.Collection, params Params) (*mgo.Query, error) {

	q, err := h.Map(params)
	if err != nil {
		return nil, err
	}

	return col.Find(q), nil
}

// Accepts a params map and returns a map for use with the mgo `find()`
// function.
func (h *Query) Map(params Params) (interface{}, error) {

	// Create the template
	t, err := template.New("haigo").Parse(h.QueryString)
	if err != nil {
		return nil, err
	}

	// Buffer to capture string
	buf := new(bytes.Buffer)

	// Execute template
	err = t.Execute(buf, sanitizeParams(params))
	if err != nil {
		return nil, err
	}

	// Unmarshal JSON into Map
	var m interface{}
	err = json.Unmarshal(buf.Bytes(), &m)
	if err != nil {
		return nil, err
	}

	return m, nil
}

// YAML formatted file with MongoDB Queries.
//
//  ---
//    - name: basic-select
//      description: Basic MongoDB Select
//      query: '{"type": {{.type}} }'
//
//    - name: conditional
//      description: Conditional Query
//      query: '{
//         "type": "food",
//         "$or": [ { "qty": { "$gt": {{.qty}} } }, { "name": {{.name}} } ]
//      }'
type File struct {
	Queries map[string]*Query
}

func (m *File) unmarshalYAML(data []byte) error {

	var hqs []Query

	err := yaml.Unmarshal(data, &hqs)
	if err != nil {
		return err
	}

	qm := make(map[string]*Query)

	for i := range hqs {
		qm[hqs[i].Name] = &hqs[i]
	}

	m.Queries = qm

	return nil
}

// sanitizeParams - Adds single quotes if param is a string (as needed by Mongo).
func sanitizeParams(params Params) Params {
	for k, v := range params {
		switch v.(type) {
		case string:
			params[k] = fmt.Sprintf("\"%s\"", v)
		}

	}
	return params
}

// Reads in Mongo Query File for use with Haigo.
func LoadQueryFile(file string) (*File, error) {
	return parseMongoFile(file)
}
<commit_msg>remove sanitize params<commit_after>package haigo

import (
	"bytes"
	"encoding/json"
	"text/template"

	"gopkg.in/mgo.v2"
	"gopkg.in/yaml.v2"
)

type param struct {
	Name string
	Type string
}

type Query struct {
	Name        string  `yaml:"name"`
	Description string  `yaml:"description,omitempty"`
	QueryString string  `yaml:"query"`
	params      []param // TODO
}

type Params map[string]interface{}

// Returns configured mgo Pipe.
func (h *Query) Pipe(col *mgo.Collection, params Params) (*mgo.Pipe, error) {

	q, err := h.Map(params)
	if err != nil {
		return nil, err
	}

	return col.Pipe(q), nil
}

// Returns configured mgo Query.
func (h *Query) Query(col *mgo.Collection, params Params) (*mgo.Query, error) {

	q, err := h.Map(params)
	if err != nil {
		return nil, err
	}

	return col.Find(q), nil
}

// Accepts a params map and returns a map for use with the mgo `find()`
// function.
func (h *Query) Map(params Params) (interface{}, error) {

	// Create the template
	t, err := template.New("haigo").Parse(h.QueryString)
	if err != nil {
		return nil, err
	}

	// Buffer to capture string
	buf := new(bytes.Buffer)

	// Execute template
	err = t.Execute(buf, params)
	if err != nil {
		return nil, err
	}

	// Unmarshal JSON into Map
	var m interface{}
	err = json.Unmarshal(buf.Bytes(), &m)
	if err != nil {
		return nil, err
	}

	return m, nil
}

// YAML formatted file with MongoDB Queries.
//
//  ---
//    - name: basic-select
//      description: Basic MongoDB Select
//      query: '{"type": {{.type}} }'
//
//    - name: conditional
//      description: Conditional Query
//      query: '{
//         "type": "food",
//         "$or": [ { "qty": { "$gt": {{.qty}} } }, { "name": {{.name}} } ]
//      }'
type File struct {
	Queries map[string]*Query
}

func (m *File) unmarshalYAML(data []byte) error {

	var hqs []Query

	err := yaml.Unmarshal(data, &hqs)
	if err != nil {
		return err
	}

	qm := make(map[string]*Query)

	for i := range hqs {
		qm[hqs[i].Name] = &hqs[i]
	}

	m.Queries = qm

	return nil
}

// Reads in Mongo Query File for use with Haigo.
func LoadQueryFile(file string) (*File, error) {
	return parseMongoFile(file)
}
<|endoftext|>


<commit_before>// Copyright © 2016 Jacob Dearing
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package havoc

import (
	"crypto/rand"
	"runtime/debug"
)

var Data = make([]byte, 0)

// SetMemory sets the exported Data byte array to a given size
func SetMemory(size int) {
	Data = make([]byte, size)
}

// FreeMemory forces VM to release unused memory back to the system
func FreeMemory() {
	debug.FreeOSMemory()
}

// ResetMemory calls Setmemory(0) followed by FreeMemory
func ResetMemory() {
	SetMemory(0)
	FreeMemory()
}

// FillData will fill the current Data array with random data
func FillData() {
	rand.Read(Data)
}

func Forever() {
	for {

	}
}
<commit_msg>comment Forever<commit_after>// Copyright © 2016 Jacob Dearing
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package havoc

import (
	"crypto/rand"
	"runtime/debug"
)

var Data = make([]byte, 0)

// SetMemory sets the exported Data byte array to a given size
func SetMemory(size int) {
	Data = make([]byte, size)
}

// FreeMemory forces VM to release unused memory back to the system
func FreeMemory() {
	debug.FreeOSMemory()
}

// ResetMemory calls Setmemory(0) followed by FreeMemory
func ResetMemory() {
	SetMemory(0)
	FreeMemory()
}

// FillData will fill the current Data array with random data
func FillData() {
	rand.Read(Data)
}

// Forever runs forever.
func Forever() {
	for {

	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"
)

type Path struct {
	Path string
	IsDir bool
	ModTime time.Time
}

func IndexAndCompare(paths []string, excludes []string) {
	generateLocalIndex(paths)
}

func generateLocalIndex(paths []string) {
	index := make([]Path, 1, 2)
	for _, path := range paths {
		err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
			index = append(index, info)
			fmt.Println(info.ModTime())
			return err
		})
		if err != nil {
			fmt.Println(err)
		}
	}
}
<commit_msg>Removed broken struct use.<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"time"
)

type Path struct {
	Path string
	IsDir bool
	ModTime time.Time
}

func IndexAndCompare(paths []string, excludes []string) {
	generateLocalIndex(paths)
}

func generateLocalIndex(paths []string) {
	//index := make([]Path, 1, 2)
	for _, path := range paths {
		err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
			//index = append(index, time.Now())
			fmt.Println(info.ModTime())
			return err
		})
		if err != nil {
			fmt.Println(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/blevesearch/bleve"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

func loadOffer(store *Store, id string) (*jsonOffer, error) {
	data, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	offer := &jsonOffer{}
	err = json.Unmarshal(data, offer)
	return offer, err
}

type offerResult struct {
	Id    string
	Offer *jsonOffer
	Err   error
}

func loadOffers(store *Store) ([]*jsonOffer, error) {
	ids := store.List()
	sort.Strings(ids)
	pending := make(chan string, len(ids))
	for _, id := range ids {
		pending <- id
	}
	close(pending)

	results := make(chan offerResult, len(ids))
	running := &sync.WaitGroup{}
	jobs := 4
	for i := 0; i < jobs; i++ {
		running.Add(1)
		go func() {
			defer running.Done()
			for id := range pending {
				offer, err := loadOffer(store, id)
				results <- offerResult{
					Id:    id,
					Offer: offer,
					Err:   err,
				}
			}
		}()
	}
	go func() {
		running.Wait()
		close(results)
	}()

	offers := []*jsonOffer{}
	for r := range results {
		if r.Err != nil {
			fmt.Printf("loading error for %s: %s\n", r.Id, r.Err)
			continue
		}
		offers = append(offers, r.Offer)
	}
	return offers, nil
}

type Offer struct {
	Account   string
	Id        string `json:"id"`
	HTML      string `json:"html"`
	Title     string `json:"title"`
	MinSalary int    `json:"min_salary"`
	MaxSalary int    `json:"max_salary"`
	Date      time.Time
	URL       string
	Location  string `json:"location"`
	City      string `json:"city"`
	County    string `json:"county"`
	State     string `json:"state"`
	Country   string `json:"country"`
}

var (
	reSalaryNum   = regexp.MustCompile(`(\d+(?:\.\d+)?)`)
	reSalaryUndef = regexp.MustCompile(`^(?:.*(definir|negoc|profil|experience|a voir|determiner|attract|precise|selon|competitif).*|nc|-)$`)
)

func isMn(r rune) bool {
	return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
}

var (
	cleaner = transform.Chain(norm.NFD,
		transform.RemoveFunc(isMn),
		norm.NFC)
)

func normString(s string) string {
	result, _, _ := transform.String(cleaner, s)
	return result
}

func parseSalary(s string) (int, int, error) {
	s = strings.ToLower(normString(s))
	m := reSalaryNum.FindAllStringSubmatch(s, -1)
	if m != nil {
		values := []int{}
		for _, n := range m {
			v, err := strconv.ParseFloat(n[0], 32)
			if err != nil {
				return -1, -1, err
			}
			if v >= 1000 {
				v = v / 1000.
			}
			values = append(values, int(v))
		}
		switch len(values) {
		case 1:
			return values[0], values[0], nil
		case 2:
			return values[0], values[1], nil
		}
		return 0, 0, fmt.Errorf("too many numbers")
	}
	return 0, 0, nil
}

const (
	ApecURL = "https://cadres.apec.fr/offres-emploi-cadres/offre.html?numIdOffre="
)

func convertOffer(offer *jsonOffer) (*Offer, error) {
	r := &Offer{
		Account:  offer.Account,
		Id:       offer.Id,
		HTML:     offer.HTML,
		Title:    offer.Title,
		URL:      ApecURL + offer.Id,
		Location: offer.Location,
	}
	min, max, err := parseSalary(offer.Salary)
	if err != nil {
		return nil, fmt.Errorf("cannot parse salary %q: %s", offer.Salary, err)
	}
	d, err := time.Parse("2006-01-02T15:04:05.000+0000", offer.Date)
	if err != nil {
		return nil, err
	}
	r.Date = d
	r.MinSalary = min
	r.MaxSalary = max
	return r, nil
}

func convertOffers(offers []*jsonOffer) ([]*Offer, error) {
	result := make([]*Offer, 0, len(offers))
	for _, o := range offers {
		r, err := convertOffer(o)
		if err != nil {
			fmt.Printf("error: cannot parse salary %q: %s\n", o.Salary, err)
			continue
		}
		result = append(result, r)
	}
	return result, nil
}

func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.MkdirAll(dir, 0755)
	if err != nil {
		return nil, err
	}

	textAll := bleve.NewTextFieldMapping()
	textAll.Store = false
	textAll.IncludeTermVectors = false

	text := bleve.NewTextFieldMapping()
	text.Store = false
	text.IncludeInAll = false
	text.IncludeTermVectors = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", textAll)
	offer.AddFieldMappingsAt("title", textAll)
	offer.AddFieldMappingsAt("city", text)
	offer.AddFieldMappingsAt("county", text)
	offer.AddFieldMappingsAt("state", text)
	offer.AddFieldMappingsAt("country", text)

	m := bleve.NewIndexMapping()
	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.New(filepath.Join(dir, "index"), m)
	if err != nil {
		return nil, err
	}
	return index, nil
}

func fixLocation(s string) string {
	if !utf8.ValidString(s) {
		fmt.Printf("invalid: %s\n", s)
		u, _, err := transform.String(charmap.Windows1252.NewDecoder(), s)
		if err != nil {
			fmt.Printf("invalid: %s\n", s)
			return s
		}
		if s != u {
			fmt.Printf("recoded: %s => %s\n", s, u)
		}
		s = u
	}
	s = strings.TrimSpace(s)
	l := strings.ToLower(s)
	if l == "idf" {
		return "Ile-de-France"
	}
	return s
}

func geocodeOffer(geocoder *Geocoder, offer *Offer) (string, *Location, error) {
	q := fixLocation(offer.Location)
	loc, err := geocoder.Geocode(q, "fr")
	if err != nil {
		return q, nil, err
	}
	if len(loc.Results) == 0 {
		return q, loc, nil
	}
	res := loc.Results[0].Component
	offer.City = res.City
	offer.County = res.County
	offer.State = res.State
	offer.Country = res.Country
	return q, loc, nil
}

var (
	indexCmd     = app.Command("index", "index APEC offers")
	indexDataDir = indexCmd.Flag("data", "data directory").Default("offers").String()
	indexMaxSize = indexCmd.Flag("max-count", "maximum number of items to index").
			Short('n').Default("0").Int()
	indexGeocoderKey = indexCmd.Flag("geocoding-key", "geocoder API key").String()
)

func indexOffers() error {
	dirs := NewDataDirs(*indexDataDir)
	var geocoder *Geocoder
	if *indexGeocoderKey != "" {
		g, err := NewGeocoder(*indexGeocoderKey, dirs.Geocoder())
		if err != nil {
			return err
		}
		geocoder = g
		defer func() {
			if geocoder != nil {
				geocoder.Close()
			}
		}()
	}
	store, err := OpenStore(dirs.Store())
	if err != nil {
		return err
	}
	index, err := NewOfferIndex(dirs.Index())
	if err != nil {
		return err
	}
	rawOffers, err := loadOffers(store)
	if err != nil {
		return err
	}
	offers, err := convertOffers(rawOffers)
	if err != nil {
		return err
	}
	if *indexMaxSize > 0 && len(offers) > *indexMaxSize {
		offers = offers[:*indexMaxSize]
	}
	start := time.Now()
	for i, offer := range offers {
		if (i+1)%500 == 0 {
			now := time.Now()
			elapsed := float64(now.Sub(start)) / float64(time.Second)
			fmt.Printf("%d indexed, %.1f/s\n", i+1, float64(i+1)/elapsed)
		}
		if geocoder != nil {
			q, loc, err := geocodeOffer(geocoder, offer)
			if err != nil {
				fmt.Printf("error: geocoding %s: %s\n", q, err)
				if err == QuotaError {
					geocoder = nil
					break
				}
			} else if !loc.Cached {
				result := "no result"
				if len(loc.Results) > 0 {
					result = loc.Results[0].Component.String()
				}
				fmt.Printf("geocoding %s => %s (quota: %d/%d)\n", q, result,
					loc.Rate.Remaining, loc.Rate.Limit)
				time.Sleep(1 * time.Second)
			}
		}

		err = index.Index(offer.Id, offer)
		if err != nil {
			return err
		}
	}
	err = index.Close()
	if err != nil {
		return err
	}
	end := time.Now()
	fmt.Printf("%d documents indexed in %.2fs\n", len(offers),
		float64(end.Sub(start))/float64(time.Second))
	return nil
}
<commit_msg>index: delete index before rebuilding it<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
	"unicode"
	"unicode/utf8"

	"github.com/blevesearch/bleve"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

func loadOffer(store *Store, id string) (*jsonOffer, error) {
	data, err := store.Get(id)
	if err != nil {
		return nil, err
	}
	offer := &jsonOffer{}
	err = json.Unmarshal(data, offer)
	return offer, err
}

type offerResult struct {
	Id    string
	Offer *jsonOffer
	Err   error
}

func loadOffers(store *Store) ([]*jsonOffer, error) {
	ids := store.List()
	sort.Strings(ids)
	pending := make(chan string, len(ids))
	for _, id := range ids {
		pending <- id
	}
	close(pending)

	results := make(chan offerResult, len(ids))
	running := &sync.WaitGroup{}
	jobs := 4
	for i := 0; i < jobs; i++ {
		running.Add(1)
		go func() {
			defer running.Done()
			for id := range pending {
				offer, err := loadOffer(store, id)
				results <- offerResult{
					Id:    id,
					Offer: offer,
					Err:   err,
				}
			}
		}()
	}
	go func() {
		running.Wait()
		close(results)
	}()

	offers := []*jsonOffer{}
	for r := range results {
		if r.Err != nil {
			fmt.Printf("loading error for %s: %s\n", r.Id, r.Err)
			continue
		}
		offers = append(offers, r.Offer)
	}
	return offers, nil
}

type Offer struct {
	Account   string
	Id        string `json:"id"`
	HTML      string `json:"html"`
	Title     string `json:"title"`
	MinSalary int    `json:"min_salary"`
	MaxSalary int    `json:"max_salary"`
	Date      time.Time
	URL       string
	Location  string `json:"location"`
	City      string `json:"city"`
	County    string `json:"county"`
	State     string `json:"state"`
	Country   string `json:"country"`
}

var (
	reSalaryNum   = regexp.MustCompile(`(\d+(?:\.\d+)?)`)
	reSalaryUndef = regexp.MustCompile(`^(?:.*(definir|negoc|profil|experience|a voir|determiner|attract|precise|selon|competitif).*|nc|-)$`)
)

func isMn(r rune) bool {
	return unicode.Is(unicode.Mn, r) // Mn: nonspacing marks
}

var (
	cleaner = transform.Chain(norm.NFD,
		transform.RemoveFunc(isMn),
		norm.NFC)
)

func normString(s string) string {
	result, _, _ := transform.String(cleaner, s)
	return result
}

func parseSalary(s string) (int, int, error) {
	s = strings.ToLower(normString(s))
	m := reSalaryNum.FindAllStringSubmatch(s, -1)
	if m != nil {
		values := []int{}
		for _, n := range m {
			v, err := strconv.ParseFloat(n[0], 32)
			if err != nil {
				return -1, -1, err
			}
			if v >= 1000 {
				v = v / 1000.
			}
			values = append(values, int(v))
		}
		switch len(values) {
		case 1:
			return values[0], values[0], nil
		case 2:
			return values[0], values[1], nil
		}
		return 0, 0, fmt.Errorf("too many numbers")
	}
	return 0, 0, nil
}

const (
	ApecURL = "https://cadres.apec.fr/offres-emploi-cadres/offre.html?numIdOffre="
)

func convertOffer(offer *jsonOffer) (*Offer, error) {
	r := &Offer{
		Account:  offer.Account,
		Id:       offer.Id,
		HTML:     offer.HTML,
		Title:    offer.Title,
		URL:      ApecURL + offer.Id,
		Location: offer.Location,
	}
	min, max, err := parseSalary(offer.Salary)
	if err != nil {
		return nil, fmt.Errorf("cannot parse salary %q: %s", offer.Salary, err)
	}
	d, err := time.Parse("2006-01-02T15:04:05.000+0000", offer.Date)
	if err != nil {
		return nil, err
	}
	r.Date = d
	r.MinSalary = min
	r.MaxSalary = max
	return r, nil
}

func convertOffers(offers []*jsonOffer) ([]*Offer, error) {
	result := make([]*Offer, 0, len(offers))
	for _, o := range offers {
		r, err := convertOffer(o)
		if err != nil {
			fmt.Printf("error: cannot parse salary %q: %s\n", o.Salary, err)
			continue
		}
		result = append(result, r)
	}
	return result, nil
}

func NewOfferIndex(dir string) (bleve.Index, error) {
	err := os.RemoveAll(dir)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	err = os.MkdirAll(dir, 0755)
	if err != nil {
		return nil, err
	}

	textAll := bleve.NewTextFieldMapping()
	textAll.Store = false
	textAll.IncludeTermVectors = false

	text := bleve.NewTextFieldMapping()
	text.Store = false
	text.IncludeInAll = false
	text.IncludeTermVectors = false

	offer := bleve.NewDocumentStaticMapping()
	offer.Dynamic = false
	offer.AddFieldMappingsAt("html", textAll)
	offer.AddFieldMappingsAt("title", textAll)
	offer.AddFieldMappingsAt("city", text)
	offer.AddFieldMappingsAt("county", text)
	offer.AddFieldMappingsAt("state", text)
	offer.AddFieldMappingsAt("country", text)

	m := bleve.NewIndexMapping()
	m.AddDocumentMapping("offer", offer)
	m.DefaultMapping = offer

	index, err := bleve.New(filepath.Join(dir, "index"), m)
	if err != nil {
		return nil, err
	}
	return index, nil
}

func fixLocation(s string) string {
	if !utf8.ValidString(s) {
		fmt.Printf("invalid: %s\n", s)
		u, _, err := transform.String(charmap.Windows1252.NewDecoder(), s)
		if err != nil {
			fmt.Printf("invalid: %s\n", s)
			return s
		}
		if s != u {
			fmt.Printf("recoded: %s => %s\n", s, u)
		}
		s = u
	}
	s = strings.TrimSpace(s)
	l := strings.ToLower(s)
	if l == "idf" {
		return "Ile-de-France"
	}
	return s
}

func geocodeOffer(geocoder *Geocoder, offer *Offer) (string, *Location, error) {
	q := fixLocation(offer.Location)
	loc, err := geocoder.Geocode(q, "fr")
	if err != nil {
		return q, nil, err
	}
	if len(loc.Results) == 0 {
		return q, loc, nil
	}
	res := loc.Results[0].Component
	offer.City = res.City
	offer.County = res.County
	offer.State = res.State
	offer.Country = res.Country
	return q, loc, nil
}

var (
	indexCmd     = app.Command("index", "index APEC offers")
	indexDataDir = indexCmd.Flag("data", "data directory").Default("offers").String()
	indexMaxSize = indexCmd.Flag("max-count", "maximum number of items to index").
			Short('n').Default("0").Int()
	indexGeocoderKey = indexCmd.Flag("geocoding-key", "geocoder API key").String()
)

func indexOffers() error {
	dirs := NewDataDirs(*indexDataDir)
	var geocoder *Geocoder
	if *indexGeocoderKey != "" {
		g, err := NewGeocoder(*indexGeocoderKey, dirs.Geocoder())
		if err != nil {
			return err
		}
		geocoder = g
		defer func() {
			if geocoder != nil {
				geocoder.Close()
			}
		}()
	}
	store, err := OpenStore(dirs.Store())
	if err != nil {
		return err
	}
	index, err := NewOfferIndex(dirs.Index())
	if err != nil {
		return err
	}
	rawOffers, err := loadOffers(store)
	if err != nil {
		return err
	}
	offers, err := convertOffers(rawOffers)
	if err != nil {
		return err
	}
	if *indexMaxSize > 0 && len(offers) > *indexMaxSize {
		offers = offers[:*indexMaxSize]
	}
	start := time.Now()
	for i, offer := range offers {
		if (i+1)%500 == 0 {
			now := time.Now()
			elapsed := float64(now.Sub(start)) / float64(time.Second)
			fmt.Printf("%d indexed, %.1f/s\n", i+1, float64(i+1)/elapsed)
		}
		if geocoder != nil {
			q, loc, err := geocodeOffer(geocoder, offer)
			if err != nil {
				fmt.Printf("error: geocoding %s: %s\n", q, err)
				if err == QuotaError {
					geocoder = nil
					break
				}
			} else if !loc.Cached {
				result := "no result"
				if len(loc.Results) > 0 {
					result = loc.Results[0].Component.String()
				}
				fmt.Printf("geocoding %s => %s (quota: %d/%d)\n", q, result,
					loc.Rate.Remaining, loc.Rate.Limit)
				time.Sleep(1 * time.Second)
			}
		}

		err = index.Index(offer.Id, offer)
		if err != nil {
			return err
		}
	}
	err = index.Close()
	if err != nil {
		return err
	}
	end := time.Now()
	fmt.Printf("%d documents indexed in %.2fs\n", len(offers),
		float64(end.Sub(start))/float64(time.Second))
	return nil
}
<|endoftext|>


<commit_before>package index

import (
	"appengine"
	"appengine/urlfetch"

	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type Request struct {
	ReqId    string   `json:"reqId"`
	Dates    []string `json:"dates"`
	TimeZone string   `json:"timeZone"`
}

func (request *Request) newRequest() (r *Request) {
	r = request
	newDates := make([]string, 0)

	tz := request.TimeZone
	location, _ := time.LoadLocation(tz)
	today := time.Now().In(location)

	length := len(request.Dates)
	for i := 0; i < length; i++ {
		t, err := time.Parse("2006-1-2", request.Dates[i])

		//Check for invalid dateformat.
		if err != nil || t.Year() > today.Year() || t.Year() < 1998 || t.Year() == 0 || int(t.Month()) == 0 || t.Day() <= 0 {
			continue
		}

		//Check for invalid "day".
		ss := strings.Split(request.Dates[i], "-")
		if day, err := strconv.Atoi(ss[2]); err == nil {
			if day <= 0 {
				continue
			}
		} else {
			continue
		}

		//Check for invalid in "this month".
		if t.Year() == today.Year() && t.Month() > today.Month() {
			continue
		} else {
			newDates = append(newDates, request.Dates[i])
		}
	}
	r.Dates = newDates
	return
}

type LastThreeRequest struct {
	ReqId    string `json:"reqId"`
	TimeZone string `json:"timeZone"`
}

func (request *LastThreeRequest) newRequest() (r *Request) {
	tz := request.TimeZone
	location, _ := time.LoadLocation(tz)

	today := time.Now().In(location)
	now := today.Format("2006-1-2")
	beforeYesterday := today.AddDate(0, 0, -2).Format("2006-1-2")
	yesterday := today.AddDate(0, 0, -1).Format("2006-1-2")
	r = &Request{request.ReqId, []string{beforeYesterday, yesterday, now}, request.TimeZone}
	return
}

type MonthRequest struct {
	ReqId    string `json:"reqId"`
	Year     int    `json:"year"`
	Month    int    `json:"month"`
	TimeZone string `json:"timeZone"`
}

// daysIn returns the number of days in a month for a given year.
func (request *MonthRequest) daysIn() int {
	// This is equivalent to time.daysIn(m, year).
	mon := time.Month(request.Month + 1)
	return time.Date(request.Year, mon, 0, 0, 0, 0, 0, time.UTC).Day()
}

func (request *MonthRequest) newRequest() (r *Request) {
	tz := request.TimeZone
	location, _ := time.LoadLocation(tz)
	today := time.Now().In(location)

	r = &Request{}
	r.ReqId = request.ReqId
	r.Dates = []string{}

	//Check for invalid year, zero objects.
	if request.Year < 1998 || request.Year > today.Year() || request.Year == 0 || request.Month == 0 {
		return
	}

	//Check for invalid month in this year.
	if request.Year == today.Year() && request.Month > int(today.Month()) {
		return
	}

	daysIn := request.daysIn()
	mon := time.Month(request.Month)
	for day := 1; day <= daysIn; day++ {
		//Check for invalid day in "this month".
		if request.Year == today.Year() && request.Month == int(today.Month()) {
			if day > today.Day() {
				return
			}
		}
		dt := time.Date(request.Year, mon, day, 0, 0, 0, 0, time.UTC)
		r.Dates = append(r.Dates, dt.Format("2006-1-2"))
	}
	return
}

type Meta struct {
	Title       string `json:"title"`
	Explanation string `json:"explanation"`
	Date        string `json:"date"`
	Url         string `json:"url"`
	HDUrl       string `json:"hdurl"`
	MediaType   string `json:"media_type"`
}

type Photo struct {
	ReqId       string `json:"reqId"`
	Title       string `json:"title"`
	Description string `json:"description"`
	Date        string `json:"date"`
	Urls        Urls   `json:"urls"`
	Type        string `json:"type"`
}

func (photo *Photo) fromMeta(pMeta *Meta) {
	photo.Title = pMeta.Title
	photo.Description = pMeta.Explanation
	photo.Date = pMeta.Date
	photo.Urls = Urls{pMeta.Url, pMeta.HDUrl}
	photo.Type = pMeta.MediaType
	photo.ReqId = pMeta.Date
}

type Urls struct {
	Normal string `json:"normal"`
	HD     string `json:"hd"`
}

type Error string

func (e Error) Error() string {
	return string(e)
}

func init() {
	http.HandleFunc("/list", handleList)
	http.HandleFunc("/last_three_list", handleLastThreeList)
	http.HandleFunc("/month_list", handleMonthList)
}

func status(w http.ResponseWriter, reqId string, status int) {
	if reqId == "" {
		reqId = "not provided"
	}
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, `{"status":%d, "message" : "%s"}`, status, reqId)
}

func response(w http.ResponseWriter, reqId string, photo []*Photo) {
	if reqId == "" {
		reqId = "not provided"
	}
	json, _ := json.Marshal(photo)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, `{"status":200, "reqId" : "%s", "result" : %s}`, reqId, string(json))
}

//getPhoto
//Call API of NASA to getting photos.
func getPhoto(r *http.Request, pDate *string, chPhoto chan *Photo) {
	cxt := appengine.NewContext(r)
	url := fmt.Sprintf(API_APOD, HOST, *pDate)
	if req, err := http.NewRequest("GET", url, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pMeta := new(Meta)
				json.Unmarshal(bytes, pMeta)
				photo := new(Photo)
				photo.fromMeta(pMeta)
				chPhoto <- photo
			} else {
				cxt.Errorf("getPhoto: %v", err)
				chPhoto <- nil
			}
		} else {
			cxt.Errorf("getPhoto: %v", err)
			chPhoto <- nil
		}
	} else {
		cxt.Errorf("getPhoto: %v", err)
		chPhoto <- nil
	}
}

func buildResult(r *http.Request, p *Request) (list []*Photo) {
	dates := p.Dates
	length := len(dates)
	list = make([]*Photo, 0)
	ch := make(chan *Photo, length)
	for i := 0; i < length; i++ {
		dt := dates[i]
		go getPhoto(r, &dt, ch)
		v := <-ch
		if v != nil {
			list = append(list, v)
		}
	}
	return
}

//showList
func showList(w http.ResponseWriter, r *http.Request, p *Request) {
	plist := buildResult(r, p)
	if len(plist) > 0 {
		response(w, p.ReqId, plist)
	} else {
		status(w, "Empty result.", 300)
	}
}

//handleList
//Get list of photos of specified dates.
func handleList(w http.ResponseWriter, r *http.Request) {
	defer func() {
		if err := recover(); err != nil {
			status(w, fmt.Sprintf("%v", err), 500)
		}
	}()

	req := Request{}
	if bytes, e := ioutil.ReadAll(r.Body); e == nil {
		if e := json.Unmarshal(bytes, &req); e == nil {
			showList(w, r, req.newRequest())
		} else {
			s := fmt.Sprintf("Body content is invalid: %v", e)
			status(w, s, 302)
		}
	} else {
		s := fmt.Sprintf("Body to the request is invalid: %v", e)
		status(w, s, 301)
	}
}

//handleLastThreeList
//Get list for last three days including today.
func handleLastThreeList(w http.ResponseWriter, r *http.Request) {
	defer func() {
		if err := recover(); err != nil {
			status(w, fmt.Sprintf("%v", err), 500)
		}
	}()

	ltr := LastThreeRequest{}
	if bytes, e := ioutil.ReadAll(r.Body); e == nil {
		if e := json.Unmarshal(bytes, &ltr); e == nil {
			showList(w, r, ltr.newRequest())
		} else {
			s := fmt.Sprintf("Body content is invalid: %v", e)
			status(w, s, 302)
		}
	} else {
		s := fmt.Sprintf("Body to the request is invalid: %v", e)
		status(w, s, 301)
	}
}

//handleMonthList
//Get list for whole month.
func handleMonthList(w http.ResponseWriter, r *http.Request) {
	defer func() {
		if err := recover(); err != nil {
			status(w, fmt.Sprintf("%v", err), 500)
		}
	}()

	monthRequest := MonthRequest{}
	if bytes, e := ioutil.ReadAll(r.Body); e == nil {
		if e := json.Unmarshal(bytes, &monthRequest); e == nil {
			showList(w, r, monthRequest.newRequest())
		} else {
			s := fmt.Sprintf("Body content is invalid: %v", e)
			status(w, s, 302)
		}
	} else {
		s := fmt.Sprintf("Body to the request is invalid: %v", e)
		status(w, s, 301)
	}
}
<commit_msg>Ignore empty photo<commit_after>package index

import (
	"appengine"
	"appengine/urlfetch"

	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
	"time"
)

type Request struct {
	ReqId    string   `json:"reqId"`
	Dates    []string `json:"dates"`
	TimeZone string   `json:"timeZone"`
}

func (request *Request) newRequest() (r *Request) {
	r = request
	newDates := make([]string, 0)

	tz := request.TimeZone
	location, _ := time.LoadLocation(tz)
	today := time.Now().In(location)

	length := len(request.Dates)
	for i := 0; i < length; i++ {
		t, err := time.Parse("2006-1-2", request.Dates[i])

		//Check for invalid dateformat.
		if err != nil || t.Year() > today.Year() || t.Year() < 1998 || t.Year() == 0 || int(t.Month()) == 0 || t.Day() <= 0 {
			continue
		}

		//Check for invalid "day".
		ss := strings.Split(request.Dates[i], "-")
		if day, err := strconv.Atoi(ss[2]); err == nil {
			if day <= 0 {
				continue
			}
		} else {
			continue
		}

		//Check for invalid in "this month".
		if t.Year() == today.Year() && t.Month() > today.Month() {
			continue
		} else {
			newDates = append(newDates, request.Dates[i])
		}
	}
	r.Dates = newDates
	return
}

type LastThreeRequest struct {
	ReqId    string `json:"reqId"`
	TimeZone string `json:"timeZone"`
}

func (request *LastThreeRequest) newRequest() (r *Request) {
	tz := request.TimeZone
	location, _ := time.LoadLocation(tz)

	today := time.Now().In(location)
	now := today.Format("2006-1-2")
	beforeYesterday := today.AddDate(0, 0, -2).Format("2006-1-2")
	yesterday := today.AddDate(0, 0, -1).Format("2006-1-2")
	r = &Request{request.ReqId, []string{beforeYesterday, yesterday, now}, request.TimeZone}
	return
}

type MonthRequest struct {
	ReqId    string `json:"reqId"`
	Year     int    `json:"year"`
	Month    int    `json:"month"`
	TimeZone string `json:"timeZone"`
}

// daysIn returns the number of days in a month for a given year.
func (request *MonthRequest) daysIn() int {
	// This is equivalent to time.daysIn(m, year).
	mon := time.Month(request.Month + 1)
	return time.Date(request.Year, mon, 0, 0, 0, 0, 0, time.UTC).Day()
}

func (request *MonthRequest) newRequest() (r *Request) {
	tz := request.TimeZone
	location, _ := time.LoadLocation(tz)
	today := time.Now().In(location)

	r = &Request{}
	r.ReqId = request.ReqId
	r.Dates = []string{}

	//Check for invalid year, zero objects.
	if request.Year < 1998 || request.Year > today.Year() || request.Year == 0 || request.Month == 0 {
		return
	}

	//Check for invalid month in this year.
	if request.Year == today.Year() && request.Month > int(today.Month()) {
		return
	}

	daysIn := request.daysIn()
	mon := time.Month(request.Month)
	for day := 1; day <= daysIn; day++ {
		//Check for invalid day in "this month".
		if request.Year == today.Year() && request.Month == int(today.Month()) {
			if day > today.Day() {
				return
			}
		}
		dt := time.Date(request.Year, mon, day, 0, 0, 0, 0, time.UTC)
		r.Dates = append(r.Dates, dt.Format("2006-1-2"))
	}
	return
}

type Meta struct {
	Title       string `json:"title"`
	Explanation string `json:"explanation"`
	Date        string `json:"date"`
	Url         string `json:"url"`
	HDUrl       string `json:"hdurl"`
	MediaType   string `json:"media_type"`
}

type Photo struct {
	ReqId       string `json:"reqId"`
	Title       string `json:"title"`
	Description string `json:"description"`
	Date        string `json:"date"`
	Urls        Urls   `json:"urls"`
	Type        string `json:"type"`
}

func (photo *Photo) fromMeta(pMeta *Meta) {
	photo.Title = pMeta.Title
	photo.Description = pMeta.Explanation
	photo.Date = pMeta.Date
	photo.Urls = Urls{pMeta.Url, pMeta.HDUrl}
	photo.Type = pMeta.MediaType
	photo.ReqId = pMeta.Date
}

type Urls struct {
	Normal string `json:"normal"`
	HD     string `json:"hd"`
}

type Error string

func (e Error) Error() string {
	return string(e)
}

func init() {
	http.HandleFunc("/list", handleList)
	http.HandleFunc("/last_three_list", handleLastThreeList)
	http.HandleFunc("/month_list", handleMonthList)
}

func status(w http.ResponseWriter, reqId string, status int) {
	if reqId == "" {
		reqId = "not provided"
	}
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, `{"status":%d, "message" : "%s"}`, status, reqId)
}

func response(w http.ResponseWriter, reqId string, photo []*Photo) {
	if reqId == "" {
		reqId = "not provided"
	}
	json, _ := json.Marshal(photo)
	w.Header().Set("Content-Type", "application/json")
	fmt.Fprintf(w, `{"status":200, "reqId" : "%s", "result" : %s}`, reqId, string(json))
}

//getPhoto
//Call API of NASA to getting photos.
func getPhoto(r *http.Request, pDate *string, chPhoto chan *Photo) {
	cxt := appengine.NewContext(r)
	url := fmt.Sprintf(API_APOD, HOST, *pDate)
	if req, err := http.NewRequest("GET", url, nil); err == nil {
		httpClient := urlfetch.Client(cxt)
		r, err := httpClient.Do(req)
		if r != nil {
			defer r.Body.Close()
		}
		if err == nil {
			if bytes, err := ioutil.ReadAll(r.Body); err == nil {
				pMeta := new(Meta)
				json.Unmarshal(bytes, pMeta)
				photo := new(Photo)
				photo.fromMeta(pMeta)
				if photo.Title == "" {
					chPhoto <- nil
				} else {
					chPhoto <- photo
				}
			} else {
				cxt.Errorf("getPhoto: %v", err)
				chPhoto <- nil
			}
		} else {
			cxt.Errorf("getPhoto: %v", err)
			chPhoto <- nil
		}
	} else {
		cxt.Errorf("getPhoto: %v", err)
		chPhoto <- nil
	}
}

func buildResult(r *http.Request, p *Request) (list []*Photo) {
	dates := p.Dates
	length := len(dates)
	list = make([]*Photo, 0)
	ch := make(chan *Photo, length)
	for i := 0; i < length; i++ {
		dt := dates[i]
		go getPhoto(r, &dt, ch)
		v := <-ch
		if v != nil {
			list = append(list, v)
		}
	}
	return
}

//showList
func showList(w http.ResponseWriter, r *http.Request, p *Request) {
	plist := buildResult(r, p)
	if len(plist) > 0 {
		response(w, p.ReqId, plist)
	} else {
		status(w, "Empty result.", 300)
	}
}

//handleList
//Get list of photos of specified dates.
func handleList(w http.ResponseWriter, r *http.Request) {
	defer func() {
		if err := recover(); err != nil {
			status(w, fmt.Sprintf("%v", err), 500)
		}
	}()

	req := Request{}
	if bytes, e := ioutil.ReadAll(r.Body); e == nil {
		if e := json.Unmarshal(bytes, &req); e == nil {
			showList(w, r, req.newRequest())
		} else {
			s := fmt.Sprintf("Body content is invalid: %v", e)
			status(w, s, 302)
		}
	} else {
		s := fmt.Sprintf("Body to the request is invalid: %v", e)
		status(w, s, 301)
	}
}

//handleLastThreeList
//Get list for last three days including today.
func handleLastThreeList(w http.ResponseWriter, r *http.Request) {
	defer func() {
		if err := recover(); err != nil {
			status(w, fmt.Sprintf("%v", err), 500)
		}
	}()

	ltr := LastThreeRequest{}
	if bytes, e := ioutil.ReadAll(r.Body); e == nil {
		if e := json.Unmarshal(bytes, &ltr); e == nil {
			showList(w, r, ltr.newRequest())
		} else {
			s := fmt.Sprintf("Body content is invalid: %v", e)
			status(w, s, 302)
		}
	} else {
		s := fmt.Sprintf("Body to the request is invalid: %v", e)
		status(w, s, 301)
	}
}

//handleMonthList
//Get list for whole month.
func handleMonthList(w http.ResponseWriter, r *http.Request) {
	defer func() {
		if err := recover(); err != nil {
			status(w, fmt.Sprintf("%v", err), 500)
		}
	}()

	monthRequest := MonthRequest{}
	if bytes, e := ioutil.ReadAll(r.Body); e == nil {
		if e := json.Unmarshal(bytes, &monthRequest); e == nil {
			showList(w, r, monthRequest.newRequest())
		} else {
			s := fmt.Sprintf("Body content is invalid: %v", e)
			status(w, s, 302)
		}
	} else {
		s := fmt.Sprintf("Body to the request is invalid: %v", e)
		status(w, s, 301)
	}
}
<|endoftext|>