<commit_before>// web100 provides Go bindings to some functions in the web100 library.
package web100

// Cgo directives must immediately preceed 'import "C"' below.
// For more information see:
//  - https://blog.golang.org/c-go-cgo
//  - https://golang.org/cmd/cgo

/*
#include <stdio.h>
#include <stdlib.h>
#include <web100.h>
#include <web100-int.h>

#include <arpa/inet.h>
*/
import "C"

import (
	"fmt"
	"io"
	"log"
	"net"
	"strconv"
	"sync"
	"unsafe"
)

var (
	// TODO(prod): eliminate this lock (along with tmpfs).
	web100Lock sync.Mutex
)

// Discoveries:
//  - Not all C macros exist in the "C" namespace.
//  - 'NULL' is usually equivalent to 'nil'

// The Saver interface decouples reading data from the web100 log files and
// saving those values.
type Saver interface {
	SetInt64(name string, value int64)
	SetString(name string, value string)
}

// Web100 maintains state associated with a web100 log file.
type Web100 struct {
	// legacyNames maps legacy web100 variable names to their canonical names.
	legacyNames map[string]string

	// NOTE: we define all C-allocated types as unsafe.Pointers here. This is a
	// design choice to emphasize that these values should not be used outside
	// of this package and even within this package, they should be used
	// carefully.

	// snaplog is the primary *C.web100_log object encapsulating a snaplog file.
	snaplog unsafe.Pointer

	// snap is an individual *C.web100_snapshot record read from a snaplog.
	snap unsafe.Pointer

	// Pointers to C buffers for use in calls to web100 functions.
	text unsafe.Pointer
	data unsafe.Pointer
}

// Open prepares a web100 snaplog file for reading. The caller must call Close on
// the returned Web100 instance to free memory and close open file descriptors.
func Open(filename string, legacyNames map[string]string) (*Web100, error) {
	c_filename := C.CString(filename)
	defer C.free(unsafe.Pointer(c_filename))

	// TODO(prod): do not require reading from a file. Accept a byte array.
	// We need to lock calls to web100_log_open_read because of "log_header".
	var w_errno C.int = C.WEB100_ERR_SUCCESS
	web100Lock.Lock()
	snaplog := C.web100_log_open_read(c_filename, &w_errno)
	web100Lock.Unlock()
	if w_errno != C.WEB100_ERR_SUCCESS {
		fmt.Printf("%v\n", snaplog)
	}

	// Verify that the snaplog is valid before continuing.
	if snaplog == nil {
		return nil, fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
	}
	if w_errno != C.WEB100_ERR_SUCCESS {
		C.web100_log_close_read(snaplog)
		return nil, fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
	}

	// Pre-allocate a snapshot record.
	snap := C.web100_snapshot_alloc_from_log(snaplog, &w_errno)
	if snap == nil {
		log.Printf("%s\n", C.GoString(C.web100_strerror(w_errno)))
		C.web100_log_close_read(snaplog)
		return nil, fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
	}
	if w_errno != C.WEB100_ERR_SUCCESS {
		C.web100_snapshot_free(snap)
		C.web100_log_close_read(snaplog)
		return nil, fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
	}

	w := &Web100{
		legacyNames: legacyNames,
		snaplog:     unsafe.Pointer(snaplog),
		snap:        unsafe.Pointer(snap),
		// Pre-allocate space for converting snapshot values.
		text: C.calloc(2*C.WEB100_VALUE_LEN_MAX, 1),
		data: C.calloc(C.WEB100_VALUE_LEN_MAX, 1),
	}
	return w, nil
}

// Next reads the next C.web100_snapshot record from the web100 snaplog and
// saves it in an internal buffer. Use SnapshotValues to read the all values from
// the most recently read snapshot.  If Next reaches EOF or another error, the
// last snapshot is in an undefined state.
func (w *Web100) Next() error {
	snaplog := (*C.web100_log)(w.snaplog)
	snap := (*C.web100_snapshot)(w.snap)
	if snap == nil {
		log.Printf("Snapshot is nil\n")
		return fmt.Errorf("Snapshot is nil")
	}

	// Read the next web100_snaplog data from underlying file.
	err := C.web100_snap_from_log(snap, snaplog)
	if err == C.EOF {
		return io.EOF
	}
	if err != C.WEB100_ERR_SUCCESS {
		// WEB100_ERR_FILE_TRUNCATED_SNAP_DATA or
		// WEB100_ERR__MISSING_SNAP_MAGIC
		return fmt.Errorf(C.GoString(C.web100_strerror(err)))
	}
	return nil
}

// LogVersion returns the snaplog version.
func (w *Web100) LogVersion() string {
	snaplog := (*C.web100_log)(w.snaplog)
	return C.GoString(C.web100_get_agent_version(C.web100_get_log_agent(snaplog)))
}

// LogTime returns the timestamp of when the snaplog was opened for writing.
func (w *Web100) LogTime() int64 {
	snaplog := (*C.web100_log)(w.snaplog)
	return int64(C.web100_get_log_time(snaplog))
}

// ConnectionSpec populates the connSpec Saver with values from C.web100_connection_spec.
// TODO(dev): define the field names saved in connSpec since they are not the
// same ones defined in C.web100_connection_spec.
func (w *Web100) ConnectionSpec(connSpec Saver) error {
	snaplog := (*C.web100_log)(w.snaplog)

	// NOTE: web100_connection_spec_v6 is never set by the web100 library.
	// NOTE: conn->addrtype is always WEB100_ADDRTYPE_UNKNOWN.
	// So, we expect it to be IPv4 and fix local_ip and remote_ip later if
	// snapshots have IPv6 addresses.
	var spec C.struct_web100_connection_spec

	// Get reference to the snaplog connection.
	conn := C.web100_get_log_connection(snaplog)
	// Copy the connection spec from the snaplog connection information.
	C.web100_get_connection_spec(conn, &spec)

	// NOTE: web100_connection_spec only contains IPv4 addresses (4 byte values).
	// If the connection was IPv6, the IPv4 addresses here will be 0.0.0.0.
	srcIp := net.IP(C.GoBytes(unsafe.Pointer(&spec.src_addr), 4))
	dstIp := net.IP(C.GoBytes(unsafe.Pointer(&spec.dst_addr), 4))

	connSpec.SetString("local_ip", srcIp.String())
	connSpec.SetInt64("local_port", int64(spec.src_port))
	connSpec.SetString("remote_ip", dstIp.String())
	connSpec.SetInt64("remote_port", int64(spec.dst_port))
	// NOTE: legacy values of local_af are: IPv4 = 0, IPv6 = 1.
	connSpec.SetInt64("local_af", 0)

	return nil
}

// SnapshotValues saves all values from the most recent C.web100_snapshot read by
// Next. Next must be called at least once before calling SnapshotValues.
func (w *Web100) SnapshotValues(snapValues Saver) error {
	snaplog := (*C.web100_log)(w.snaplog)
	snap := (*C.web100_snapshot)(w.snap)

	// Parses variables from most recent web100_snapshot data.
	var w_errno C.int = C.WEB100_ERR_SUCCESS
	group := C.web100_get_log_group(snaplog)

	// The web100 group is a set of web100 variables from a specific agent.
	// M-Lab snaplogs only ever have a single agent ("local") and group
	// (whatever the static set of web100 variables read from
	// /proc/web100/header).
	//
	// To extract each web100 variables corresponding to all the variables
	// in the group, we iterate through each one.
	for v := C.web100_var_head(group, &w_errno); v != nil; v = C.web100_var_next(v, &w_errno) {
		if w_errno != C.WEB100_ERR_SUCCESS {
			return fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
		}

		// Extract the web100 variable name and type. This will
		// correspond to one of the variables defined in tcp-kis.txt.
		name := C.web100_get_var_name(v)
		var_type := C.web100_get_var_type(v)

		// Read the raw bytes for the variable from the snapshot.
		errno := C.web100_snap_read(v, snap, w.data)
		if errno != C.WEB100_ERR_SUCCESS {
			return fmt.Errorf(C.GoString(C.web100_strerror(errno)))
		}

		// Convert raw w.data into a string based on var_type.
		// TODO(prod): reimplement web100_value_to_textn to operate on Go types.
		C.web100_value_to_textn((*C.char)(w.text), C.WEB100_VALUE_LEN_MAX,
			(C.WEB100_TYPE)(var_type), w.data)

		// Use the canonical variable name. The variable name known to
		// the web100 kernel at run time lagged behind the official
		// web100 spec. So, some variable names need to be translated
		// from their legacy form (read from the kernel and written to
		// the snaplog) to the canonical form (as defined in
		// tcp-kis.txt).
		canonicalName := C.GoString(name)
		if _, ok := w.legacyNames[canonicalName]; ok {
			canonicalName = w.legacyNames[canonicalName]
		}

		// TODO(dev): are there any cases where we need unsigned int64?
		// Attempt to convert the current variable to an int64.
		value, err := strconv.ParseInt(C.GoString((*C.char)(w.text)), 10, 64)
		if err != nil {
			e := err.(*strconv.NumError)
			if e.Err == strconv.ErrSyntax {
				// If it cannot be converted, leave the variable as a string.
				snapValues.SetString(canonicalName, C.GoString((*C.char)(w.text)))
			} else if e.Err == strconv.ErrRange {
				log.Println("Range error: " + e.Num)
				snapValues.SetInt64(canonicalName, value)
			}
		} else {
			snapValues.SetInt64(canonicalName, value)
		}
	}
	return nil
}

// Close releases resources created by Open.
func (w *Web100) Close() error {
	snap := (*C.web100_snapshot)(w.snap)
	C.web100_snapshot_free(snap)

	snaplog := (*C.web100_log)(w.snaplog)
	err := C.web100_log_close_read(snaplog)
	if err != C.WEB100_ERR_SUCCESS {
		return fmt.Errorf(C.GoString(C.web100_strerror(err)))
	}
	if w.text != nil {
		C.free(w.text)
		w.text = nil
	}
	if w.data != nil {
		C.free(w.data)
		w.data = nil
	}

	// Clear pointer after free.
	w.snaplog = nil
	w.snap = nil
	return nil
}
<commit_msg>Add comment about range error<commit_after>// web100 provides Go bindings to some functions in the web100 library.
package web100

// Cgo directives must immediately preceed 'import "C"' below.
// For more information see:
//  - https://blog.golang.org/c-go-cgo
//  - https://golang.org/cmd/cgo

/*
#include <stdio.h>
#include <stdlib.h>
#include <web100.h>
#include <web100-int.h>

#include <arpa/inet.h>
*/
import "C"

import (
	"fmt"
	"io"
	"log"
	"net"
	"strconv"
	"sync"
	"unsafe"
)

var (
	// TODO(prod): eliminate this lock (along with tmpfs).
	web100Lock sync.Mutex
)

// Discoveries:
//  - Not all C macros exist in the "C" namespace.
//  - 'NULL' is usually equivalent to 'nil'

// The Saver interface decouples reading data from the web100 log files and
// saving those values.
type Saver interface {
	SetInt64(name string, value int64)
	SetString(name string, value string)
}

// Web100 maintains state associated with a web100 log file.
type Web100 struct {
	// legacyNames maps legacy web100 variable names to their canonical names.
	legacyNames map[string]string

	// NOTE: we define all C-allocated types as unsafe.Pointers here. This is a
	// design choice to emphasize that these values should not be used outside
	// of this package and even within this package, they should be used
	// carefully.

	// snaplog is the primary *C.web100_log object encapsulating a snaplog file.
	snaplog unsafe.Pointer

	// snap is an individual *C.web100_snapshot record read from a snaplog.
	snap unsafe.Pointer

	// Pointers to C buffers for use in calls to web100 functions.
	text unsafe.Pointer
	data unsafe.Pointer
}

// Open prepares a web100 snaplog file for reading. The caller must call Close on
// the returned Web100 instance to free memory and close open file descriptors.
func Open(filename string, legacyNames map[string]string) (*Web100, error) {
	c_filename := C.CString(filename)
	defer C.free(unsafe.Pointer(c_filename))

	// TODO(prod): do not require reading from a file. Accept a byte array.
	// We need to lock calls to web100_log_open_read because of "log_header".
	var w_errno C.int = C.WEB100_ERR_SUCCESS
	web100Lock.Lock()
	snaplog := C.web100_log_open_read(c_filename, &w_errno)
	web100Lock.Unlock()
	if w_errno != C.WEB100_ERR_SUCCESS {
		fmt.Printf("%v\n", snaplog)
	}

	// Verify that the snaplog is valid before continuing.
	if snaplog == nil {
		return nil, fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
	}
	if w_errno != C.WEB100_ERR_SUCCESS {
		C.web100_log_close_read(snaplog)
		return nil, fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
	}

	// Pre-allocate a snapshot record.
	snap := C.web100_snapshot_alloc_from_log(snaplog, &w_errno)
	if snap == nil {
		log.Printf("%s\n", C.GoString(C.web100_strerror(w_errno)))
		C.web100_log_close_read(snaplog)
		return nil, fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
	}
	if w_errno != C.WEB100_ERR_SUCCESS {
		C.web100_snapshot_free(snap)
		C.web100_log_close_read(snaplog)
		return nil, fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
	}

	w := &Web100{
		legacyNames: legacyNames,
		snaplog:     unsafe.Pointer(snaplog),
		snap:        unsafe.Pointer(snap),
		// Pre-allocate space for converting snapshot values.
		text: C.calloc(2*C.WEB100_VALUE_LEN_MAX, 1),
		data: C.calloc(C.WEB100_VALUE_LEN_MAX, 1),
	}
	return w, nil
}

// Next reads the next C.web100_snapshot record from the web100 snaplog and
// saves it in an internal buffer. Use SnapshotValues to read the all values from
// the most recently read snapshot.  If Next reaches EOF or another error, the
// last snapshot is in an undefined state.
func (w *Web100) Next() error {
	snaplog := (*C.web100_log)(w.snaplog)
	snap := (*C.web100_snapshot)(w.snap)
	if snap == nil {
		log.Printf("Snapshot is nil\n")
		return fmt.Errorf("Snapshot is nil")
	}

	// Read the next web100_snaplog data from underlying file.
	err := C.web100_snap_from_log(snap, snaplog)
	if err == C.EOF {
		return io.EOF
	}
	if err != C.WEB100_ERR_SUCCESS {
		// WEB100_ERR_FILE_TRUNCATED_SNAP_DATA or
		// WEB100_ERR__MISSING_SNAP_MAGIC
		return fmt.Errorf(C.GoString(C.web100_strerror(err)))
	}
	return nil
}

// LogVersion returns the snaplog version.
func (w *Web100) LogVersion() string {
	snaplog := (*C.web100_log)(w.snaplog)
	return C.GoString(C.web100_get_agent_version(C.web100_get_log_agent(snaplog)))
}

// LogTime returns the timestamp of when the snaplog was opened for writing.
func (w *Web100) LogTime() int64 {
	snaplog := (*C.web100_log)(w.snaplog)
	return int64(C.web100_get_log_time(snaplog))
}

// ConnectionSpec populates the connSpec Saver with values from C.web100_connection_spec.
// TODO(dev): define the field names saved in connSpec since they are not the
// same ones defined in C.web100_connection_spec.
func (w *Web100) ConnectionSpec(connSpec Saver) error {
	snaplog := (*C.web100_log)(w.snaplog)

	// NOTE: web100_connection_spec_v6 is never set by the web100 library.
	// NOTE: conn->addrtype is always WEB100_ADDRTYPE_UNKNOWN.
	// So, we expect it to be IPv4 and fix local_ip and remote_ip later if
	// snapshots have IPv6 addresses.
	var spec C.struct_web100_connection_spec

	// Get reference to the snaplog connection.
	conn := C.web100_get_log_connection(snaplog)
	// Copy the connection spec from the snaplog connection information.
	C.web100_get_connection_spec(conn, &spec)

	// NOTE: web100_connection_spec only contains IPv4 addresses (4 byte values).
	// If the connection was IPv6, the IPv4 addresses here will be 0.0.0.0.
	srcIp := net.IP(C.GoBytes(unsafe.Pointer(&spec.src_addr), 4))
	dstIp := net.IP(C.GoBytes(unsafe.Pointer(&spec.dst_addr), 4))

	connSpec.SetString("local_ip", srcIp.String())
	connSpec.SetInt64("local_port", int64(spec.src_port))
	connSpec.SetString("remote_ip", dstIp.String())
	connSpec.SetInt64("remote_port", int64(spec.dst_port))
	// NOTE: legacy values of local_af are: IPv4 = 0, IPv6 = 1.
	connSpec.SetInt64("local_af", 0)

	return nil
}

// SnapshotValues saves all values from the most recent C.web100_snapshot read by
// Next. Next must be called at least once before calling SnapshotValues.
func (w *Web100) SnapshotValues(snapValues Saver) error {
	snaplog := (*C.web100_log)(w.snaplog)
	snap := (*C.web100_snapshot)(w.snap)

	// Parses variables from most recent web100_snapshot data.
	var w_errno C.int = C.WEB100_ERR_SUCCESS
	group := C.web100_get_log_group(snaplog)

	// The web100 group is a set of web100 variables from a specific agent.
	// M-Lab snaplogs only ever have a single agent ("local") and group
	// (whatever the static set of web100 variables read from
	// /proc/web100/header).
	//
	// To extract each web100 variables corresponding to all the variables
	// in the group, we iterate through each one.
	for v := C.web100_var_head(group, &w_errno); v != nil; v = C.web100_var_next(v, &w_errno) {
		if w_errno != C.WEB100_ERR_SUCCESS {
			return fmt.Errorf(C.GoString(C.web100_strerror(w_errno)))
		}

		// Extract the web100 variable name and type. This will
		// correspond to one of the variables defined in tcp-kis.txt.
		name := C.web100_get_var_name(v)
		var_type := C.web100_get_var_type(v)

		// Read the raw bytes for the variable from the snapshot.
		errno := C.web100_snap_read(v, snap, w.data)
		if errno != C.WEB100_ERR_SUCCESS {
			return fmt.Errorf(C.GoString(C.web100_strerror(errno)))
		}

		// Convert raw w.data into a string based on var_type.
		// TODO(prod): reimplement web100_value_to_textn to operate on Go types.
		C.web100_value_to_textn((*C.char)(w.text), C.WEB100_VALUE_LEN_MAX,
			(C.WEB100_TYPE)(var_type), w.data)

		// Use the canonical variable name. The variable name known to
		// the web100 kernel at run time lagged behind the official
		// web100 spec. So, some variable names need to be translated
		// from their legacy form (read from the kernel and written to
		// the snaplog) to the canonical form (as defined in
		// tcp-kis.txt).
		canonicalName := C.GoString(name)
		if _, ok := w.legacyNames[canonicalName]; ok {
			canonicalName = w.legacyNames[canonicalName]
		}

		// TODO(dev): are there any cases where we need unsigned int64?
		// Attempt to convert the current variable to an int64.
		value, err := strconv.ParseInt(C.GoString((*C.char)(w.text)), 10, 64)
		if err != nil {
			e := err.(*strconv.NumError)
			if e.Err == strconv.ErrSyntax {
				// If it cannot be converted, leave the variable as a string.
				snapValues.SetString(canonicalName, C.GoString((*C.char)(w.text)))
			} else if e.Err == strconv.ErrRange {
				log.Println("Range error: " + e.Num)
				// On a range error, ParseInt returns the best legal value,
				// i.e., MaxInt64, or MinInt64, so we just use that value.
				snapValues.SetInt64(canonicalName, value)
			}
		} else {
			snapValues.SetInt64(canonicalName, value)
		}
	}
	return nil
}

// Close releases resources created by Open.
func (w *Web100) Close() error {
	snap := (*C.web100_snapshot)(w.snap)
	C.web100_snapshot_free(snap)

	snaplog := (*C.web100_log)(w.snaplog)
	err := C.web100_log_close_read(snaplog)
	if err != C.WEB100_ERR_SUCCESS {
		return fmt.Errorf(C.GoString(C.web100_strerror(err)))
	}
	if w.text != nil {
		C.free(w.text)
		w.text = nil
	}
	if w.data != nil {
		C.free(w.data)
		w.data = nil
	}

	// Clear pointer after free.
	w.snaplog = nil
	w.snap = nil
	return nil
}
<|endoftext|>


<commit_before>package people

import (
	"encoding/json"

	"github.com/Financial-Times/neo-utils-go"
	"github.com/jmcvetta/neoism"
)

type service struct {
	cypherRunner neoutils.CypherRunner
	indexManager neoutils.IndexManager
}

// NewCypherPeopleService provides functions for create, update, delete operations on people in Neo4j,
// plus other utility functions needed for a service
func NewCypherPeopleService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager) service {
	return service{cypherRunner, indexManager}
}

func (s service) Initialise() error {
	return neoutils.EnsureConstraints(s.indexManager, map[string]string{
		"Thing":   "uuid",
		"Concept": "uuid",
		"Person":  "uuid"})
}

func (s service) Read(uuid string) (interface{}, bool, error) {
	results := []struct {
		UUID              string   `json:"uuid"`
		Name              string   `json:"name"`
		BirthYear         int      `json:"birthYear"`
		Salutation        string   `json:"salutation"`
		FactsetIdentifier string   `json:"factsetIdentifier"`
		TMEIdentifiers    []string `json:"tmeIdentifiers"`
		Aliases           []string `json:"aliases"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Person {uuid:{uuid}}) return n.uuid
		as uuid, n.name as name,
		n.factsetIdentifier as factsetIdentifier,
		n.tmeIdentifiers as tmeIdentifiers,
		n.birthYear as birthYear,
		n.salutation as salutation,
		n.aliases as aliases`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return person{}, false, err
	}

	if len(results) == 0 {
		return person{}, false, nil
	}

	result := results[0]

	p := person{
		UUID:       result.UUID,
		Name:       result.Name,
		BirthYear:  result.BirthYear,
		Salutation: result.Salutation,
		Aliases:    result.Aliases,
	}

	if result.FactsetIdentifier != "" {
		p.Identifiers = append(p.Identifiers, identifier{fsAuthority, result.FactsetIdentifier})
	}

	if len(result.TMEIdentifiers) > 0 {
		for _, tmeValue := range result.TMEIdentifiers {
			p.Identifiers = append(p.Identifiers, identifier{tmeAuthority, tmeValue})
		}
	}

	return p, true, nil

}

func (s service) Write(thing interface{}) error {

	p := thing.(person)

	params := map[string]interface{}{
		"uuid": p.UUID,
	}

	if p.Name != "" {
		params["name"] = p.Name
		params["prefLabel"] = p.Name
	}

	if p.BirthYear != 0 {
		params["birthYear"] = p.BirthYear
	}

	if p.Salutation != "" {
		params["salutation"] = p.Salutation
	}

	var tmeIdentifiers []string

	for _, identifier := range p.Identifiers {
		if identifier.Authority == fsAuthority {
			params["factsetIdentifier"] = identifier.IdentifierValue
		}
		if identifier.Authority == tmeAuthority {
			tmeIdentifiers = append(tmeIdentifiers, identifier.IdentifierValue)
		}
	}

	if len(tmeIdentifiers) > 0 {
		params["tmeIdentifiers"] = tmeIdentifiers
	}

	var aliases []string

	for _, alias := range p.Aliases {
		aliases = append(aliases, alias)
	}

	if len(aliases) > 0 {
		params["aliases"] = aliases
	}

	query := &neoism.CypherQuery{
		Statement: `MERGE (n:Thing {uuid: {uuid}})
					set n={allprops}
					set n :Concept
					set n :Person
		`,
		Parameters: map[string]interface{}{
			"uuid":     p.UUID,
			"allprops": params,
		},
	}

	return s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

}

func (s service) Delete(uuid string) (bool, error) {
	clearNode := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			REMOVE p:Concept
			REMOVE p:Person
			SET p={props}
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
			"props": map[string]interface{}{
				"uuid": uuid,
			},
		},
		IncludeStats: true,
	}

	removeNodeIfUnused := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			OPTIONAL MATCH (p)-[a]-(x)
			WITH p, count(a) AS relCount
			WHERE relCount = 0
			DELETE p
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{clearNode, removeNodeIfUnused})

	s1, err := clearNode.Stats()
	if err != nil {
		return false, err
	}

	var deleted bool
	if s1.ContainsUpdates && s1.LabelsRemoved > 0 {
		deleted = true
	}

	return deleted, err
}

func (s service) DecodeJSON(dec *json.Decoder) (interface{}, string, error) {
	p := person{}
	err := dec.Decode(&p)
	return p, p.UUID, err
}

func (s service) Check() error {
	return neoutils.Check(s.cypherRunner)
}

func (s service) Count() (int, error) {

	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Person) return count(n) as c`,
		Result:    &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

const (
	fsAuthority = "http://api.ft.com/system/FACTSET-PPL"
)

const (
	tmeAuthority = "http://api.ft.com/system/TME"
)
<commit_msg>small refactor of uneccessary if statement<commit_after>package people

import (
	"encoding/json"

	"github.com/Financial-Times/neo-utils-go"
	"github.com/jmcvetta/neoism"
)

type service struct {
	cypherRunner neoutils.CypherRunner
	indexManager neoutils.IndexManager
}

// NewCypherPeopleService provides functions for create, update, delete operations on people in Neo4j,
// plus other utility functions needed for a service
func NewCypherPeopleService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager) service {
	return service{cypherRunner, indexManager}
}

func (s service) Initialise() error {
	return neoutils.EnsureConstraints(s.indexManager, map[string]string{
		"Thing":   "uuid",
		"Concept": "uuid",
		"Person":  "uuid"})
}

func (s service) Read(uuid string) (interface{}, bool, error) {
	results := []struct {
		UUID              string   `json:"uuid"`
		Name              string   `json:"name"`
		BirthYear         int      `json:"birthYear"`
		Salutation        string   `json:"salutation"`
		FactsetIdentifier string   `json:"factsetIdentifier"`
		TMEIdentifiers    []string `json:"tmeIdentifiers"`
		Aliases           []string `json:"aliases"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Person {uuid:{uuid}}) return n.uuid
		as uuid, n.name as name,
		n.factsetIdentifier as factsetIdentifier,
		n.tmeIdentifiers as tmeIdentifiers,
		n.birthYear as birthYear,
		n.salutation as salutation,
		n.aliases as aliases`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
		Result: &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return person{}, false, err
	}

	if len(results) == 0 {
		return person{}, false, nil
	}

	result := results[0]

	p := person{
		UUID:       result.UUID,
		Name:       result.Name,
		BirthYear:  result.BirthYear,
		Salutation: result.Salutation,
		Aliases:    result.Aliases,
	}

	if result.FactsetIdentifier != "" {
		p.Identifiers = append(p.Identifiers, identifier{fsAuthority, result.FactsetIdentifier})
	}

	// if len(result.TMEIdentifiers) > 0 {
	// 	for _, tmeValue := range result.TMEIdentifiers {
	// 		p.Identifiers = append(p.Identifiers, identifier{tmeAuthority, tmeValue})
	// 	}
	// }

	for _, tmeValue := range result.TMEIdentifiers {
		p.Identifiers = append(p.Identifiers, identifier{tmeAuthority, tmeValue})
	}

	return p, true, nil

}

func (s service) Write(thing interface{}) error {

	p := thing.(person)

	params := map[string]interface{}{
		"uuid": p.UUID,
	}

	if p.Name != "" {
		params["name"] = p.Name
		params["prefLabel"] = p.Name
	}

	if p.BirthYear != 0 {
		params["birthYear"] = p.BirthYear
	}

	if p.Salutation != "" {
		params["salutation"] = p.Salutation
	}

	var tmeIdentifiers []string

	for _, identifier := range p.Identifiers {
		if identifier.Authority == fsAuthority {
			params["factsetIdentifier"] = identifier.IdentifierValue
		}
		if identifier.Authority == tmeAuthority {
			tmeIdentifiers = append(tmeIdentifiers, identifier.IdentifierValue)
		}
	}

	if len(tmeIdentifiers) > 0 {
		params["tmeIdentifiers"] = tmeIdentifiers
	}

	var aliases []string

	for _, alias := range p.Aliases {
		aliases = append(aliases, alias)
	}

	if len(aliases) > 0 {
		params["aliases"] = aliases
	}

	query := &neoism.CypherQuery{
		Statement: `MERGE (n:Thing {uuid: {uuid}})
					set n={allprops}
					set n :Concept
					set n :Person
		`,
		Parameters: map[string]interface{}{
			"uuid":     p.UUID,
			"allprops": params,
		},
	}

	return s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

}

func (s service) Delete(uuid string) (bool, error) {
	clearNode := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			REMOVE p:Concept
			REMOVE p:Person
			SET p={props}
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
			"props": map[string]interface{}{
				"uuid": uuid,
			},
		},
		IncludeStats: true,
	}

	removeNodeIfUnused := &neoism.CypherQuery{
		Statement: `
			MATCH (p:Thing {uuid: {uuid}})
			OPTIONAL MATCH (p)-[a]-(x)
			WITH p, count(a) AS relCount
			WHERE relCount = 0
			DELETE p
		`,
		Parameters: map[string]interface{}{
			"uuid": uuid,
		},
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{clearNode, removeNodeIfUnused})

	s1, err := clearNode.Stats()
	if err != nil {
		return false, err
	}

	var deleted bool
	if s1.ContainsUpdates && s1.LabelsRemoved > 0 {
		deleted = true
	}

	return deleted, err
}

func (s service) DecodeJSON(dec *json.Decoder) (interface{}, string, error) {
	p := person{}
	err := dec.Decode(&p)
	return p, p.UUID, err
}

func (s service) Check() error {
	return neoutils.Check(s.cypherRunner)
}

func (s service) Count() (int, error) {

	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH (n:Person) return count(n) as c`,
		Result:    &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

const (
	fsAuthority = "http://api.ft.com/system/FACTSET-PPL"
)

const (
	tmeAuthority = "http://api.ft.com/system/TME"
)
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
	"net/http"
	"net/http/pprof"

	"github.com/gorilla/mux"
)

func attachProfiler(router *mux.Router) {
	router.HandleFunc("/debug/pprof/", pprof.Index)
	router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	router.HandleFunc("/debug/pprof/profile", pprof.Profile)
	router.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	for i := 1; i < 1000; i++ {
		math.Pow10(i)
	}
	fmt.Fprint(w, "Hello! Gopher")
}

func main() {
	r := mux.NewRouter()
	attachProfiler(r)
	r.HandleFunc("/", indexHandler)
	http.ListenAndServe(":8080", r)
}
<commit_msg>minor modification<commit_after>package main

import (
	"fmt"
	"math"
	"net/http"
	"net/http/pprof"

	"github.com/gorilla/mux"
)

func attachProfiler(router *mux.Router) {
	router.HandleFunc("/debug/pprof/", pprof.Index)
	router.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
	router.HandleFunc("/debug/pprof/profile", pprof.Profile)
	router.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
	router.HandleFunc("/debug/pprof/trace", pprof.Trace)
}

func indexHandler(w http.ResponseWriter, r *http.Request) {
	for i := 1; i < 1000; i++ {
		math.Pow10(i)
	}
	fmt.Fprint(w, "Hello! Gopher")
}

func main() {
	r := mux.NewRouter()
	attachProfiler(r)
	r.HandleFunc("/", indexHandler)
	http.ListenAndServe(":8080", r)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package matcher

import (
	"errors"
	"sync"
	"time"

	"github.com/m3db/m3/src/cluster/kv"
	"github.com/m3db/m3/src/cluster/kv/util/runtime"
	"github.com/m3db/m3/src/metrics/aggregation"
	"github.com/m3db/m3/src/metrics/generated/proto/rulepb"
	"github.com/m3db/m3/src/metrics/metric"
	"github.com/m3db/m3/src/metrics/rules"
	"github.com/m3db/m3/src/x/clock"
	"github.com/m3db/m3/src/x/watch"

	"github.com/uber-go/tally"
	"go.uber.org/zap"
)

var (
	emptyNamespaces rules.Namespaces
	errNilValue     = errors.New("nil value received")
)

// Namespaces manages runtime updates to registered namespaces and provides
// API to match metic ids against rules in the corresponding namespaces.
type Namespaces interface {
	// Open opens the namespaces and starts watching runtime rule updates
	Open() error

	// Version returns the current version for a give namespace.
	Version(namespace []byte) int

	// ForwardMatch forward matches the matching policies for a given id in a given namespace
	// between [fromNanos, toNanos).
	ForwardMatch(namespace, id []byte, fromNanos, toNanos int64) rules.MatchResult

	// ReverseMatch reverse matches the matching policies for a given id in a given namespace
	// between [fromNanos, toNanos), taking into account the metric type and aggregation type for the given id.
	ReverseMatch(
		namespace, id []byte,
		fromNanos, toNanos int64,
		mt metric.Type,
		at aggregation.Type,
		isMultiAggregationTypesAllowed bool,
		aggTypesOpts aggregation.TypesOptions,
	) rules.MatchResult

	// Close closes the namespaces.
	Close()
}

type rulesNamespace rules.Namespace

type namespacesMetrics struct {
	notExists         tally.Counter
	added             tally.Counter
	removed           tally.Counter
	watched           tally.Counter
	watchErrors       tally.Counter
	unwatched         tally.Counter
	createWatchErrors tally.Counter
	initWatchErrors   tally.Counter
}

func newNamespacesMetrics(scope tally.Scope) namespacesMetrics {
	return namespacesMetrics{
		notExists:         scope.Counter("not-exists"),
		added:             scope.Counter("added"),
		removed:           scope.Counter("removed"),
		watched:           scope.Counter("watched"),
		watchErrors:       scope.Counter("watch-errors"),
		unwatched:         scope.Counter("unwatched"),
		createWatchErrors: scope.Counter("create-watch-errors"),
		initWatchErrors:   scope.Counter("init-watch-errors"),
	}
}

// namespaces contains the list of namespace users have defined rules for.
type namespaces struct {
	sync.RWMutex
	runtime.Value

	key                  string
	store                kv.Store
	opts                 Options
	nowFn                clock.NowFn
	log                  *zap.Logger
	ruleSetKeyFn         RuleSetKeyFn
	matchRangePast       time.Duration
	onNamespaceAddedFn   OnNamespaceAddedFn
	onNamespaceRemovedFn OnNamespaceRemovedFn

	proto   *rulepb.Namespaces
	rules   *namespaceRuleSetsMap
	metrics namespacesMetrics
}

// NewNamespaces creates a new namespaces object.
func NewNamespaces(key string, opts Options) Namespaces {
	instrumentOpts := opts.InstrumentOptions()
	n := &namespaces{
		key:                  key,
		store:                opts.KVStore(),
		opts:                 opts,
		nowFn:                opts.ClockOptions().NowFn(),
		log:                  instrumentOpts.Logger(),
		ruleSetKeyFn:         opts.RuleSetKeyFn(),
		matchRangePast:       opts.MatchRangePast(),
		onNamespaceAddedFn:   opts.OnNamespaceAddedFn(),
		onNamespaceRemovedFn: opts.OnNamespaceRemovedFn(),
		proto:                &rulepb.Namespaces{},
		rules:                newNamespaceRuleSetsMap(namespaceRuleSetsMapOptions{}),
		metrics:              newNamespacesMetrics(instrumentOpts.MetricsScope()),
	}
	valueOpts := runtime.NewOptions().
		SetInstrumentOptions(instrumentOpts).
		SetInitWatchTimeout(opts.InitWatchTimeout()).
		SetKVStore(n.store).
		SetUnmarshalFn(n.toNamespaces).
		SetProcessFn(n.process)
	n.Value = runtime.NewValue(key, valueOpts)
	return n
}

func (n *namespaces) Open() error {
	err := n.Watch()
	if err == nil {
		return nil
	}

	errCreateWatch, ok := err.(watch.CreateWatchError)
	if ok {
		n.metrics.createWatchErrors.Inc(1)
		return errCreateWatch
	}
	// NB(xichen): we managed to watch the key but weren't able
	// to initialize the value. In this case, log the error instead
	// to be more resilient to error conditions preventing process
	// from starting up.
	n.metrics.initWatchErrors.Inc(1)
	n.opts.InstrumentOptions().Logger().With(
		zap.String("key", n.key),
		zap.Error(err),
	).Error("error initializing namespaces values, retrying in the background")
	return nil
}

func (n *namespaces) Version(namespace []byte) int {
	n.RLock()
	ruleSet, exists := n.rules.Get(namespace)
	n.RUnlock()
	if !exists {
		return kv.UninitializedVersion
	}
	return ruleSet.Version()
}

func (n *namespaces) ForwardMatch(namespace, id []byte, fromNanos, toNanos int64) rules.MatchResult {
	ruleSet, exists := n.ruleSet(namespace)
	if !exists {
		return rules.EmptyMatchResult
	}
	return ruleSet.ForwardMatch(id, fromNanos, toNanos)
}

func (n *namespaces) ReverseMatch(
	namespace, id []byte,
	fromNanos, toNanos int64,
	mt metric.Type,
	at aggregation.Type,
	isMultiAggregationTypesAllowed bool,
	aggTypesOpts aggregation.TypesOptions,
) rules.MatchResult {
	ruleSet, exists := n.ruleSet(namespace)
	if !exists {
		return rules.EmptyMatchResult
	}
	return ruleSet.ReverseMatch(id, fromNanos, toNanos, mt, at, isMultiAggregationTypesAllowed, aggTypesOpts)
}

func (n *namespaces) ruleSet(namespace []byte) (RuleSet, bool) {
	n.RLock()
	ruleSet, exists := n.rules.Get(namespace)
	n.RUnlock()
	if !exists {
		n.metrics.notExists.Inc(1)
	}
	return ruleSet, exists
}

func (n *namespaces) Close() {
	// NB(xichen): we stop watching the value outside lock because otherwise we might
	// be holding the namespace lock while attempting to acquire the value lock, and
	// the updating goroutine might be holding the value lock and attempting to
	// acquire the namespace lock, causing a deadlock.
	n.Value.Unwatch()

	n.RLock()
	for _, entry := range n.rules.Iter() {
		rs := entry.Value()
		rs.Unwatch()
	}
	n.RUnlock()
}

func (n *namespaces) toNamespaces(value kv.Value) (interface{}, error) {
	n.Lock()
	defer n.Unlock()

	if value == nil {
		return emptyNamespaces, errNilValue
	}
	n.proto.Reset()
	if err := value.Unmarshal(n.proto); err != nil {
		return emptyNamespaces, err
	}
	return rules.NewNamespaces(value.Version(), n.proto)
}

func (n *namespaces) process(value interface{}) error {
	var (
		nss        = value.(rules.Namespaces)
		version    = nss.Version()
		namespaces = nss.Namespaces()
		incoming   = newRuleNamespacesMap(ruleNamespacesMapOptions{
			InitialSize: len(namespaces),
		})
	)
	for _, ns := range namespaces {
		incoming.Set(ns.Name(), rulesNamespace(ns))
	}

	n.Lock()
	defer n.Unlock()

	for _, entry := range incoming.Iter() {
		namespace, elem := entry.Key(), rules.Namespace(entry.Value())
		nsName, snapshots := elem.Name(), elem.Snapshots()
		ruleSet, exists := n.rules.Get(namespace)
		if !exists {
			instrumentOpts := n.opts.InstrumentOptions()
			ruleSetScope := instrumentOpts.MetricsScope().SubScope("ruleset")
			ruleSetOpts := n.opts.SetInstrumentOptions(instrumentOpts.SetMetricsScope(ruleSetScope))
			ruleSetKey := n.ruleSetKeyFn(elem.Name())
			ruleSet = newRuleSet(nsName, ruleSetKey, ruleSetOpts)
			n.rules.Set(namespace, ruleSet)
			n.metrics.added.Inc(1)
		}

		shouldWatch := true
		// This should never happen but just to be on the defensive side.
		if len(snapshots) == 0 {
			n.log.Warn("namespace updates have no snapshots", zap.Int("version", version))
		} else {
			latestSnapshot := snapshots[len(snapshots)-1]
			// If the latest update shows the namespace is tombstoned, and we
			// have received the corresponding ruleset update, we can stop watching
			// the ruleset updates.
			if latestSnapshot.Tombstoned() && latestSnapshot.ForRuleSetVersion() == ruleSet.Version() {
				shouldWatch = false
			}
		}
		if !shouldWatch {
			n.metrics.unwatched.Inc(1)
			ruleSet.Unwatch()
		} else {
			n.metrics.watched.Inc(1)
			if err := ruleSet.Watch(); err != nil {
				n.metrics.watchErrors.Inc(1)
				n.log.Error("failed to watch ruleset updates",
					zap.String("ruleSetKey", ruleSet.Key()),
					zap.Error(err))
			}
		}

		if !exists && n.onNamespaceAddedFn != nil {
			n.onNamespaceAddedFn(nsName, ruleSet)
		}
	}

	for _, entry := range n.rules.Iter() {
		namespace, ruleSet := entry.Key(), entry.Value()
		_, exists := incoming.Get(namespace)
		if exists {
			continue
		}
		// Process the namespaces not in the incoming update.
		earliestNanos := n.nowFn().Add(-n.matchRangePast).UnixNano()
		if ruleSet.Tombstoned() && ruleSet.CutoverNanos() <= earliestNanos {
			if n.onNamespaceRemovedFn != nil {
				n.onNamespaceRemovedFn(ruleSet.Namespace())
			}
			n.rules.Delete(namespace)
			ruleSet.Unwatch()
			n.metrics.unwatched.Inc(1)
		}
	}

	return nil
}
<commit_msg>watch rules async (#1556)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package matcher

import (
	"errors"
	"sync"
	"time"

	"github.com/m3db/m3/src/cluster/kv"
	"github.com/m3db/m3/src/cluster/kv/util/runtime"
	"github.com/m3db/m3/src/metrics/aggregation"
	"github.com/m3db/m3/src/metrics/generated/proto/rulepb"
	"github.com/m3db/m3/src/metrics/metric"
	"github.com/m3db/m3/src/metrics/rules"
	"github.com/m3db/m3/src/x/clock"
	"github.com/m3db/m3/src/x/watch"

	"github.com/uber-go/tally"
	"go.uber.org/zap"
)

var (
	emptyNamespaces rules.Namespaces
	errNilValue     = errors.New("nil value received")
)

// Namespaces manages runtime updates to registered namespaces and provides
// API to match metic ids against rules in the corresponding namespaces.
type Namespaces interface {
	// Open opens the namespaces and starts watching runtime rule updates
	Open() error

	// Version returns the current version for a give namespace.
	Version(namespace []byte) int

	// ForwardMatch forward matches the matching policies for a given id in a given namespace
	// between [fromNanos, toNanos).
	ForwardMatch(namespace, id []byte, fromNanos, toNanos int64) rules.MatchResult

	// ReverseMatch reverse matches the matching policies for a given id in a given namespace
	// between [fromNanos, toNanos), taking into account the metric type and aggregation type for the given id.
	ReverseMatch(
		namespace, id []byte,
		fromNanos, toNanos int64,
		mt metric.Type,
		at aggregation.Type,
		isMultiAggregationTypesAllowed bool,
		aggTypesOpts aggregation.TypesOptions,
	) rules.MatchResult

	// Close closes the namespaces.
	Close()
}

type rulesNamespace rules.Namespace

type namespacesMetrics struct {
	notExists         tally.Counter
	added             tally.Counter
	removed           tally.Counter
	watched           tally.Counter
	watchErrors       tally.Counter
	unwatched         tally.Counter
	createWatchErrors tally.Counter
	initWatchErrors   tally.Counter
}

func newNamespacesMetrics(scope tally.Scope) namespacesMetrics {
	return namespacesMetrics{
		notExists:         scope.Counter("not-exists"),
		added:             scope.Counter("added"),
		removed:           scope.Counter("removed"),
		watched:           scope.Counter("watched"),
		watchErrors:       scope.Counter("watch-errors"),
		unwatched:         scope.Counter("unwatched"),
		createWatchErrors: scope.Counter("create-watch-errors"),
		initWatchErrors:   scope.Counter("init-watch-errors"),
	}
}

// namespaces contains the list of namespace users have defined rules for.
type namespaces struct {
	sync.RWMutex
	runtime.Value

	key                  string
	store                kv.Store
	opts                 Options
	nowFn                clock.NowFn
	log                  *zap.Logger
	ruleSetKeyFn         RuleSetKeyFn
	matchRangePast       time.Duration
	onNamespaceAddedFn   OnNamespaceAddedFn
	onNamespaceRemovedFn OnNamespaceRemovedFn

	proto   *rulepb.Namespaces
	rules   *namespaceRuleSetsMap
	metrics namespacesMetrics
}

// NewNamespaces creates a new namespaces object.
func NewNamespaces(key string, opts Options) Namespaces {
	instrumentOpts := opts.InstrumentOptions()
	n := &namespaces{
		key:                  key,
		store:                opts.KVStore(),
		opts:                 opts,
		nowFn:                opts.ClockOptions().NowFn(),
		log:                  instrumentOpts.Logger(),
		ruleSetKeyFn:         opts.RuleSetKeyFn(),
		matchRangePast:       opts.MatchRangePast(),
		onNamespaceAddedFn:   opts.OnNamespaceAddedFn(),
		onNamespaceRemovedFn: opts.OnNamespaceRemovedFn(),
		proto:                &rulepb.Namespaces{},
		rules:                newNamespaceRuleSetsMap(namespaceRuleSetsMapOptions{}),
		metrics:              newNamespacesMetrics(instrumentOpts.MetricsScope()),
	}
	valueOpts := runtime.NewOptions().
		SetInstrumentOptions(instrumentOpts).
		SetInitWatchTimeout(opts.InitWatchTimeout()).
		SetKVStore(n.store).
		SetUnmarshalFn(n.toNamespaces).
		SetProcessFn(n.process)
	n.Value = runtime.NewValue(key, valueOpts)
	return n
}

func (n *namespaces) Open() error {
	err := n.Watch()
	if err == nil {
		return nil
	}

	errCreateWatch, ok := err.(watch.CreateWatchError)
	if ok {
		n.metrics.createWatchErrors.Inc(1)
		return errCreateWatch
	}
	// NB(xichen): we managed to watch the key but weren't able
	// to initialize the value. In this case, log the error instead
	// to be more resilient to error conditions preventing process
	// from starting up.
	n.metrics.initWatchErrors.Inc(1)
	n.opts.InstrumentOptions().Logger().With(
		zap.String("key", n.key),
		zap.Error(err),
	).Error("error initializing namespaces values, retrying in the background")
	return nil
}

func (n *namespaces) Version(namespace []byte) int {
	n.RLock()
	ruleSet, exists := n.rules.Get(namespace)
	n.RUnlock()
	if !exists {
		return kv.UninitializedVersion
	}
	return ruleSet.Version()
}

func (n *namespaces) ForwardMatch(namespace, id []byte, fromNanos, toNanos int64) rules.MatchResult {
	ruleSet, exists := n.ruleSet(namespace)
	if !exists {
		return rules.EmptyMatchResult
	}
	return ruleSet.ForwardMatch(id, fromNanos, toNanos)
}

func (n *namespaces) ReverseMatch(
	namespace, id []byte,
	fromNanos, toNanos int64,
	mt metric.Type,
	at aggregation.Type,
	isMultiAggregationTypesAllowed bool,
	aggTypesOpts aggregation.TypesOptions,
) rules.MatchResult {
	ruleSet, exists := n.ruleSet(namespace)
	if !exists {
		return rules.EmptyMatchResult
	}
	return ruleSet.ReverseMatch(id, fromNanos, toNanos, mt, at, isMultiAggregationTypesAllowed, aggTypesOpts)
}

func (n *namespaces) ruleSet(namespace []byte) (RuleSet, bool) {
	n.RLock()
	ruleSet, exists := n.rules.Get(namespace)
	n.RUnlock()
	if !exists {
		n.metrics.notExists.Inc(1)
	}
	return ruleSet, exists
}

func (n *namespaces) Close() {
	// NB(xichen): we stop watching the value outside lock because otherwise we might
	// be holding the namespace lock while attempting to acquire the value lock, and
	// the updating goroutine might be holding the value lock and attempting to
	// acquire the namespace lock, causing a deadlock.
	n.Value.Unwatch()

	n.RLock()
	for _, entry := range n.rules.Iter() {
		rs := entry.Value()
		rs.Unwatch()
	}
	n.RUnlock()
}

func (n *namespaces) toNamespaces(value kv.Value) (interface{}, error) {
	n.Lock()
	defer n.Unlock()

	if value == nil {
		return emptyNamespaces, errNilValue
	}
	n.proto.Reset()
	if err := value.Unmarshal(n.proto); err != nil {
		return emptyNamespaces, err
	}
	return rules.NewNamespaces(value.Version(), n.proto)
}

func (n *namespaces) process(value interface{}) error {
	var (
		nss        = value.(rules.Namespaces)
		version    = nss.Version()
		namespaces = nss.Namespaces()
		incoming   = newRuleNamespacesMap(ruleNamespacesMapOptions{
			InitialSize: len(namespaces),
		})
	)
	for _, ns := range namespaces {
		incoming.Set(ns.Name(), rulesNamespace(ns))
	}

	n.Lock()
	defer n.Unlock()

	var watchWg sync.WaitGroup
	for _, entry := range incoming.Iter() {
		namespace, elem := entry.Key(), rules.Namespace(entry.Value())
		nsName, snapshots := elem.Name(), elem.Snapshots()
		ruleSet, exists := n.rules.Get(namespace)
		if !exists {
			instrumentOpts := n.opts.InstrumentOptions()
			ruleSetScope := instrumentOpts.MetricsScope().SubScope("ruleset")
			ruleSetOpts := n.opts.SetInstrumentOptions(instrumentOpts.SetMetricsScope(ruleSetScope))
			ruleSetKey := n.ruleSetKeyFn(elem.Name())
			ruleSet = newRuleSet(nsName, ruleSetKey, ruleSetOpts)
			n.rules.Set(namespace, ruleSet)
			n.metrics.added.Inc(1)
		}

		shouldWatch := true
		// This should never happen but just to be on the defensive side.
		if len(snapshots) == 0 {
			n.log.Warn("namespace updates have no snapshots", zap.Int("version", version))
		} else {
			latestSnapshot := snapshots[len(snapshots)-1]
			// If the latest update shows the namespace is tombstoned, and we
			// have received the corresponding ruleset update, we can stop watching
			// the ruleset updates.
			if latestSnapshot.Tombstoned() && latestSnapshot.ForRuleSetVersion() == ruleSet.Version() {
				shouldWatch = false
			}
		}

		if !shouldWatch {
			n.metrics.unwatched.Inc(1)
			ruleSet.Unwatch()
		} else {
			n.metrics.watched.Inc(1)

			watchWg.Add(1)
			go func() {
				// Start the watches in background goroutines so that if the store is unavailable they timeout
				// (approximately) in unison. This prevents the timeouts from stacking on top of each
				// other when the store is unavailable and causing a delay of timeout_duration * num_rules.
				defer watchWg.Done()

				if err := ruleSet.Watch(); err != nil {
					n.metrics.watchErrors.Inc(1)
					n.log.Error("failed to watch ruleset updates",
						zap.String("ruleSetKey", ruleSet.Key()),
						zap.Error(err))
				}
			}()
		}

		if !exists && n.onNamespaceAddedFn != nil {
			n.onNamespaceAddedFn(nsName, ruleSet)
		}
	}

	watchWg.Wait()
	for _, entry := range n.rules.Iter() {
		namespace, ruleSet := entry.Key(), entry.Value()
		_, exists := incoming.Get(namespace)
		if exists {
			continue
		}
		// Process the namespaces not in the incoming update.
		earliestNanos := n.nowFn().Add(-n.matchRangePast).UnixNano()
		if ruleSet.Tombstoned() && ruleSet.CutoverNanos() <= earliestNanos {
			if n.onNamespaceRemovedFn != nil {
				n.onNamespaceRemovedFn(ruleSet.Namespace())
			}
			n.rules.Delete(namespace)
			ruleSet.Unwatch()
			n.metrics.unwatched.Inc(1)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Licensed to Elasticsearch B.V. under one or more agreements.
// Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information.

// +build integration

package esapi_test

import (
	"encoding/json"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/elastic/go-elasticsearch/v8"
)

func TestAPI(t *testing.T) {
	t.Run("Search", func(t *testing.T) {
		es, err := elasticsearch.NewDefaultClient()
		if err != nil {
			t.Fatalf("Error creating the client: %s\n", err)
		}

		es.Cluster.Health(es.Cluster.Health.WithWaitForStatus("yellow"))
		res, err := es.Search(es.Search.WithTimeout(500 * time.Millisecond))
		if err != nil {
			t.Fatalf("Error getting the response: %s\n", err)
		}
		defer res.Body.Close()

		if res.IsError() {
			t.Fatalf("Error response: %s", res.String())
		}

		var d map[string]interface{}
		err = json.NewDecoder(res.Body).Decode(&d)
		if err != nil {
			t.Fatalf("Error parsing the response: %s\n", err)
		}
		fmt.Printf("took=%vms\n", d["took"])
	})

	t.Run("Headers", func(t *testing.T) {
		es, err := elasticsearch.NewDefaultClient()
		if err != nil {
			t.Fatalf("Error creating the client: %s\n", err)
		}

		res, err := es.Info(es.Info.WithHeader(map[string]string{"Accept": "application/yaml"}))
		if err != nil {
			t.Fatalf("Error getting the response: %s\n", err)
		}
		defer res.Body.Close()

		if res.IsError() {
			t.Fatalf("Error response: %s", res.String())
		}

		if !strings.HasPrefix(res.String(), "[200 OK] ---") {
			t.Errorf("Unexpected response body: doesn't start with '[200 OK] ---'; %s", res.String())
		}
	})
}
<commit_msg>API: Add integration test for X-Opaque-ID and tasks<commit_after>// Licensed to Elasticsearch B.V. under one or more agreements.
// Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information.

// +build integration

package esapi_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/elastic/go-elasticsearch/v8"
	"github.com/elastic/go-elasticsearch/v8/esapi"
)

func TestAPI(t *testing.T) {
	t.Run("Search", func(t *testing.T) {
		es, err := elasticsearch.NewDefaultClient()
		if err != nil {
			t.Fatalf("Error creating the client: %s\n", err)
		}

		es.Cluster.Health(es.Cluster.Health.WithWaitForStatus("yellow"))
		res, err := es.Search(es.Search.WithTimeout(500 * time.Millisecond))
		if err != nil {
			t.Fatalf("Error getting the response: %s\n", err)
		}
		defer res.Body.Close()

		if res.IsError() {
			t.Fatalf("Error response: %s", res.String())
		}

		var d map[string]interface{}
		err = json.NewDecoder(res.Body).Decode(&d)
		if err != nil {
			t.Fatalf("Error parsing the response: %s\n", err)
		}
		fmt.Printf("took=%vms\n", d["took"])
	})

	t.Run("Headers", func(t *testing.T) {
		es, err := elasticsearch.NewDefaultClient()
		if err != nil {
			t.Fatalf("Error creating the client: %s\n", err)
		}

		res, err := es.Info(es.Info.WithHeader(map[string]string{"Accept": "application/yaml"}))
		if err != nil {
			t.Fatalf("Error getting the response: %s\n", err)
		}
		defer res.Body.Close()

		if res.IsError() {
			t.Fatalf("Error response: %s", res.String())
		}

		if !strings.HasPrefix(res.String(), "[200 OK] ---") {
			t.Errorf("Unexpected response body: doesn't start with '[200 OK] ---'; %s", res.String())
		}
	})

	t.Run("OpaqueID", func(t *testing.T) {
		var (
			buf bytes.Buffer

			res *esapi.Response
			err error

			requestID = "reindex-123"
		)

		es, err := elasticsearch.NewDefaultClient()
		if err != nil {
			t.Fatalf("Error creating the client: %s\n", err)
		}

		// Prepare indices
		//
		es.Indices.Delete([]string{"test", "reindexed"}, es.Indices.Delete.WithIgnoreUnavailable(true))

		// Index data
		//
		for j := 1; j <= 1000; j++ {
			meta := []byte(fmt.Sprintf(`{ "index" : { "_id" : "%d" } }%s`, j, "\n"))
			data := []byte(`{"content":"` + strings.Repeat("ABC", 100) + `"}`)
			data = append(data, "\n"...)

			buf.Grow(len(meta) + len(data))
			buf.Write(meta)
			buf.Write(data)
		}
		res, err = es.Bulk(bytes.NewReader(buf.Bytes()), es.Bulk.WithIndex("test"), es.Bulk.WithRefresh("true"))
		if err != nil {
			t.Fatalf("Failed to index data: %s", err)
		}
		res.Body.Close()
		if res.IsError() {
			t.Fatalf("Failed to index data: %s", res.Status())
		}

		// Launch reindexing task with wait_for_completion=false
		//
		res, err = es.Reindex(
			strings.NewReader(`{"source":{"index":"test"}, "dest": {"index":"reindexed"}}`),
			es.Reindex.WithWaitForCompletion(false),
			es.Reindex.WithRequestsPerSecond(1),
			es.Reindex.WithOpaqueID(requestID))
		if err != nil {
			t.Fatalf("Failed to reindex: %s", err)
		}
		if res.IsError() {
			t.Fatalf("Failed to reindex: %s", res.Status())
		}
		time.Sleep(10 * time.Millisecond)

		res, err = es.Tasks.List(es.Tasks.List.WithPretty())
		if err != nil {
			t.Fatalf("ERROR: %s", err)
		}
		res.Body.Close()
		if res.IsError() {
			t.Fatalf("Failed to get tasks: %s", res.Status())
		}

		// Get the list of tasks
		//
		res, err = es.Tasks.List(es.Tasks.List.WithPretty())
		if err != nil {
			t.Fatalf("ERROR: %s", err)
		}
		defer res.Body.Close()

		if res.IsError() {
			t.Fatalf("Failed to get tasks: %s", res.Status())
		}

		type task struct {
			Node        string
			ID          int
			Action      string
			RunningTime time.Duration `json:"running_time_in_nanos"`
			Cancellable bool
			Headers     map[string]interface{}
		}

		type node struct {
			Tasks map[string]task
		}

		var nodes map[string]map[string]node
		if err := json.NewDecoder(res.Body).Decode(&nodes); err != nil {
			t.Fatalf("Failed to decode response: %s", err)
		}

		var hasReindexTask bool

		for _, n := range nodes["nodes"] {
			for taskID, task := range n.Tasks {
				if task.Headers["X-Opaque-Id"] == requestID {
					if strings.Contains(task.Action, "reindex") {
						hasReindexTask = true
					}
					fmt.Printf("* %s, %s | %s (%s)\n", requestID, taskID, task.Action, task.RunningTime)
				}
			}
		}

		if !hasReindexTask {
			t.Errorf("Expected reindex task in %+v", nodes["nodes"])
		}

		for _, n := range nodes["nodes"] {
			for taskID, task := range n.Tasks {
				if task.Headers["X-Opaque-Id"] == requestID {
					if task.Cancellable {
						fmt.Printf("=> Closing task %s\n", taskID)
						res, err = es.Tasks.Cancel(es.Tasks.Cancel.WithTaskID(taskID))
						if err != nil {
							t.Fatalf("ERROR: %s", err)
						}
						res.Body.Close()
						if res.IsError() {
							t.Fatalf("Failed to cancel task: %s", res)
						}
					}
				}
			}
		}
	})
}
<|endoftext|>


<commit_before>package main

import "os"
import "github.com/docker/engine-api/client"
import "math/rand"
import "os/exec"
import "strconv"
import "github.com/seehuhn/mt19937"
import "time"
import "os/user"

var cli *client.Client

type ExecRequest struct {
	Image string
	Cmd []string
	SourceFileName string
}

type Judge struct {
	Code string
	Compile *ExecRequest
	Exec ExecRequest
	Time int64
	Mem  int64
	TCCount int // The number of test cases
}

type JudgeResult int

const (
	Accepted            JudgeResult = 0
	WrongAnswer         JudgeResult = 1
	CompileError        JudgeResult = 2
	TimeLimitExceeded   JudgeResult = 3
	MemoryLimitExceeded JudgeResult = 4
	RuntimeError        JudgeResult = 5
	InternalError       JudgeResult = 6
	Judging             JudgeResult = 7
	CompileTimeLimitExceeded JudgeResult = 8
	CompileMemoryLimitExceeded JudgeResult = 9
)

type JudgeStatus struct {
	Case     *string
	JR       JudgeResult
	Mem      int64
	Time     int64
	Msg      *string
}

func CreateInternalError(msg string) JudgeStatus {
	return JudgeStatus{nil, InternalError, 0, 0, &msg}
}

const BASE_RAND_STRING = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

func RandomName() string {
	rng := rand.New(mt19937.New())
	rng.Seed(time.Now().UnixNano())

	res := make([]byte, 0, 32)
	for i := 0; i < 32; i++ {
		res = append(res, BASE_RAND_STRING[rng.Intn(len(BASE_RAND_STRING))])
	}

	return string(res)
}

func (j *Judge) Run(ch chan<- JudgeStatus, tests <-chan struct {
	Name string
	In  string
	Out string
}) {
	// Close a channel to send results of judging
	defer close(ch)

	// Identity
	id := RandomName()

	// User
	_, err := exec.Command("useradd", "--no-create-home", id).Output()

	if err != nil {
		ch <- CreateInternalError("Failed to create a directory to build your code. " + err.Error())

		return
	}

	uid, err := user.Lookup(id)

	if err != nil {
		ch <- CreateInternalError("Failed to look up a user. " + err.Error())

		return
	}

	uidInt, err := strconv.ParseInt(uid.Uid, 10, 64)

	if err != nil {
		ch <- CreateInternalError("Failed to parseInt uid. " + err.Error())

		return
	}
	gidInt, err := strconv.ParseInt(uid.Gid, 10, 64)
	if err != nil {
		ch <- CreateInternalError("Failed to parseInt gid. " + err.Error())

		return
	}

	defer exec.Command("userdel", id)

	// Working Directory
	path := workingDirectory + "/" + id

	err = os.Mkdir(path, 0664)

	if err != nil {
		ch <- CreateInternalError("Failed to create a directory. " + err.Error())

		return
	}

	defer os.RemoveAll(path)

	uidInt = 0
	gidInt = 0
	err = nil//os.Chown(path, int(uidInt), int(gidInt))

	if err != nil {
		ch <- CreateInternalError("Failed to chown the directory. " + err.Error())

		return
	}

	// Source File
	fp, err := os.Create(path + "/" + j.Compile.SourceFileName)

	if err != nil {
		ch <- CreateInternalError("Failed to create source file." + err.Error())

		return
	}

	l, err := fp.Write([]byte(j.Code))

	if err != nil {
		ch <- CreateInternalError("Failed to write your code on your file. " + err.Error())

		return
	}

	if l != len(j.Code) {
		ch <- CreateInternalError("Failed to write your code on your file.")

		return
	}

	fp.Close()

	err = os.Chmod(path + "/" + j.Compile.SourceFileName, 0644)

	if err != nil {
		ch <- CreateInternalError("Failed to chmod the source file. " + err.Error())

		return
	}

	// Compile
	if j.Compile != nil {
		exe, err := NewExecutor(id, 512 * 1024 * 1024, []string{"ls", "-la", "/work"}, j.Compile.Image, []string{path + ":" + "/work"}, uid.Uid)

		if err != nil {
			ch <- CreateInternalError("Failed to create a Docker container to compile your code." + err.Error())

			return
		}

		res := exe.Run(10000, "")

		exe.Delete()
		if res.Status != ExecFinished {
			switch res.Status {
			case ExecError:
				ch <- CreateInternalError("Failed to execute a compiler." + res.Stderr)

				return
			case ExecMemoryLimitExceeded:
				ch <- JudgeStatus{JR: CompileMemoryLimitExceeded}

				return
			case ExecTimeLimitExceeded:
				ch <- JudgeStatus{JR: CompileTimeLimitExceeded}

				return
			}
		}

		if res.ExitCode != 0 {
			msg := res.Stdout + res.Stderr
			ch <- JudgeStatus{JR: CompileError, Msg: &msg}

			return
		}
	}
	return

	exe, err := NewExecutor(id, j.Mem, j.Exec.Cmd, j.Exec.Image, []string{path + ":" + "/work:ro"}, uid.Uid)

	if err != nil {
		ch <- CreateInternalError("Failed to create a Docker container to judge." + err.Error())

		return
	}

	defer exe.Delete()

	tcCounter := 0
	for tc, res := <-tests; res; tc, res = <-tests {
		res := exe.Run(j.Time, tc.In)

		if res.Status != ExecFinished {
			switch res.Status {
			case ExecError:
				msg := "Failed to execute your code." + res.Stderr
				ch <- JudgeStatus{Case: &tc.Name, JR: InternalError, Msg: &msg}
			case ExecMemoryLimitExceeded:
				ch <- JudgeStatus{Case: &tc.Name, JR: MemoryLimitExceeded}
			case ExecTimeLimitExceeded:
				ch <- JudgeStatus{Case: &tc.Name, JR: TimeLimitExceeded}
			}
		}else {
			if res.ExitCode != 0 {
				ch <- JudgeStatus{Case: &tc.Name, JR: RuntimeError}
			}else {
				if res.Stdout == tc.Out {
					ch <- JudgeStatus{Case: &tc.Name, JR: Accepted}
				}else {
					ch <- JudgeStatus{Case: &tc.Name, JR: WrongAnswer}
				}
			}
		}

		tcCounter++

		msg := strconv.FormatInt(int64(tcCounter), 10) + "/" + strconv.FormatInt(int64(j.TCCount), 10)
		ch <- JudgeStatus{JR: Judging, Msg: &msg}
	}

}
<commit_msg>Sun May 22 19:07:38 JST 2016<commit_after>package main

import "os"
import "github.com/docker/engine-api/client"
import "math/rand"
import "os/exec"
import "strconv"
import "github.com/seehuhn/mt19937"
import "time"
import "os/user"

var cli *client.Client

type ExecRequest struct {
	Image string
	Cmd []string
	SourceFileName string
}

type Judge struct {
	Code string
	Compile *ExecRequest
	Exec ExecRequest
	Time int64
	Mem  int64
	TCCount int // The number of test cases
}

type JudgeResult int

const (
	Accepted            JudgeResult = 0
	WrongAnswer         JudgeResult = 1
	CompileError        JudgeResult = 2
	TimeLimitExceeded   JudgeResult = 3
	MemoryLimitExceeded JudgeResult = 4
	RuntimeError        JudgeResult = 5
	InternalError       JudgeResult = 6
	Judging             JudgeResult = 7
	CompileTimeLimitExceeded JudgeResult = 8
	CompileMemoryLimitExceeded JudgeResult = 9
)

type JudgeStatus struct {
	Case     *string
	JR       JudgeResult
	Mem      int64
	Time     int64
	Msg      *string
}

func CreateInternalError(msg string) JudgeStatus {
	return JudgeStatus{nil, InternalError, 0, 0, &msg}
}

const BASE_RAND_STRING = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"

func RandomName() string {
	rng := rand.New(mt19937.New())
	rng.Seed(time.Now().UnixNano())

	res := make([]byte, 0, 32)
	for i := 0; i < 32; i++ {
		res = append(res, BASE_RAND_STRING[rng.Intn(len(BASE_RAND_STRING))])
	}

	return string(res)
}

func (j *Judge) Run(ch chan<- JudgeStatus, tests <-chan struct {
	Name string
	In  string
	Out string
}) {
	// Close a channel to send results of judging
	defer close(ch)

	// Identity
	id := RandomName()

	// User
	_, err := exec.Command("useradd", "--no-create-home", id).Output()

	if err != nil {
		ch <- CreateInternalError("Failed to create a directory to build your code. " + err.Error())

		return
	}

	uid, err := user.Lookup(id)

	if err != nil {
		ch <- CreateInternalError("Failed to look up a user. " + err.Error())

		return
	}

	uidInt, err := strconv.ParseInt(uid.Uid, 10, 64)

	if err != nil {
		ch <- CreateInternalError("Failed to parseInt uid. " + err.Error())

		return
	}
	gidInt, err := strconv.ParseInt(uid.Gid, 10, 64)
	if err != nil {
		ch <- CreateInternalError("Failed to parseInt gid. " + err.Error())

		return
	}

	defer exec.Command("userdel", id)

	// Working Directory
	path := workingDirectory + "/" + id

	err = os.Mkdir(path, 0664)

	if err != nil {
		ch <- CreateInternalError("Failed to create a directory. " + err.Error())

		return
	}

	defer os.RemoveAll(path)

	uidInt = uidInt *gidInt
	err = nil//os.Chown(path, int(uidInt), int(gidInt))

	if err != nil {
		ch <- CreateInternalError("Failed to chown the directory. " + err.Error())

		return
	}

	// Source File
	fp, err := os.Create(path + "/" + j.Compile.SourceFileName)

	if err != nil {
		ch <- CreateInternalError("Failed to create source file." + err.Error())

		return
	}

	l, err := fp.Write([]byte(j.Code))

	if err != nil {
		ch <- CreateInternalError("Failed to write your code on your file. " + err.Error())

		return
	}

	if l != len(j.Code) {
		ch <- CreateInternalError("Failed to write your code on your file.")

		return
	}

	fp.Close()

	err = os.Chmod(path + "/" + j.Compile.SourceFileName, 0644)

	if err != nil {
		ch <- CreateInternalError("Failed to chmod the source file. " + err.Error())

		return
	}

	// Compile
	if j.Compile != nil {
		exe, err := NewExecutor(id, 512 * 1024 * 1024, []string{"ls", "-la", "/work"}, j.Compile.Image, []string{path + ":" + "/work"}, uid.Uid)

		if err != nil {
			ch <- CreateInternalError("Failed to create a Docker container to compile your code." + err.Error())

			return
		}

		res := exe.Run(10000, "")

		exe.Delete()
		if res.Status != ExecFinished {
			switch res.Status {
			case ExecError:
				ch <- CreateInternalError("Failed to execute a compiler." + res.Stderr)

				return
			case ExecMemoryLimitExceeded:
				ch <- JudgeStatus{JR: CompileMemoryLimitExceeded}

				return
			case ExecTimeLimitExceeded:
				ch <- JudgeStatus{JR: CompileTimeLimitExceeded}

				return
			}
		}

		if res.ExitCode != 0 {
			msg := res.Stdout + res.Stderr
			ch <- JudgeStatus{JR: CompileError, Msg: &msg}

			return
		}
	}
	return

	exe, err := NewExecutor(id, j.Mem, j.Exec.Cmd, j.Exec.Image, []string{path + ":" + "/work:ro"}, uid.Uid)

	if err != nil {
		ch <- CreateInternalError("Failed to create a Docker container to judge." + err.Error())

		return
	}

	defer exe.Delete()

	tcCounter := 0
	for tc, res := <-tests; res; tc, res = <-tests {
		res := exe.Run(j.Time, tc.In)

		if res.Status != ExecFinished {
			switch res.Status {
			case ExecError:
				msg := "Failed to execute your code." + res.Stderr
				ch <- JudgeStatus{Case: &tc.Name, JR: InternalError, Msg: &msg}
			case ExecMemoryLimitExceeded:
				ch <- JudgeStatus{Case: &tc.Name, JR: MemoryLimitExceeded}
			case ExecTimeLimitExceeded:
				ch <- JudgeStatus{Case: &tc.Name, JR: TimeLimitExceeded}
			}
		}else {
			if res.ExitCode != 0 {
				ch <- JudgeStatus{Case: &tc.Name, JR: RuntimeError}
			}else {
				if res.Stdout == tc.Out {
					ch <- JudgeStatus{Case: &tc.Name, JR: Accepted}
				}else {
					ch <- JudgeStatus{Case: &tc.Name, JR: WrongAnswer}
				}
			}
		}

		tcCounter++

		msg := strconv.FormatInt(int64(tcCounter), 10) + "/" + strconv.FormatInt(int64(j.TCCount), 10)
		ch <- JudgeStatus{JR: Judging, Msg: &msg}
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	compute "google.golang.org/api/compute/v1"

	core "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apimachinery/pkg/util/wait"
	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/klog"
	"k8s.io/kubernetes/pkg/controller"
)

// InstanceIDAnnotationKey is the node annotation key where the external ID is written.
const InstanceIDAnnotationKey = "container.googleapis.com/instance_id"

var errNoMetadata = fmt.Errorf("instance did not have 'kube-labels' metadata")

type nodeAnnotator struct {
	c          clientset.Interface
	ns         corelisters.NodeLister
	hasSynced  func() bool
	queue      workqueue.RateLimitingInterface
	annotators []annotator
	// for testing
	getInstance func(nodeURL string) (*compute.Instance, error)
}

func newNodeAnnotator(client clientset.Interface, nodeInformer coreinformers.NodeInformer, cs *compute.Service) (*nodeAnnotator, error) {
	gce := compute.NewInstancesService(cs)

	// TODO(mikedanese): create a registry for the labels that GKE uses. This was
	// lifted from node_startup.go and the naming scheme is adhoc and
	// inconsistent.
	ownedKubeLabels := []string{
		"cloud.google.com/gke-nodepool",
		"cloud.google.com/gke-local-ssd",
		"cloud.google.com/gke-local-scsi-ssd",
		"cloud.google.com/gke-local-nvme-ssd",
		"cloud.google.com/gke-preemptible",
		"cloud.google.com/gke-gpu",
		"cloud.google.com/gke-accelerator",
		"beta.kubernetes.io/fluentd-ds-ready",
		"beta.kubernetes.io/kube-proxy-ds-ready",
		"beta.kubernetes.io/masq-agent-ds-ready",
		"projectcalico.org/ds-ready",
		"beta.kubernetes.io/metadata-proxy-ready",
		"addon.gke.io/node-local-dns-ds-ready",
	}

	na := &nodeAnnotator{
		c:         client,
		ns:        nodeInformer.Lister(),
		hasSynced: nodeInformer.Informer().HasSynced,
		queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewMaxOfRateLimiter(
			workqueue.NewItemExponentialFailureRateLimiter(200*time.Millisecond, 1000*time.Second),
		), "node-annotator"),
		getInstance: func(nodeURL string) (*compute.Instance, error) {
			project, zone, instance, err := parseNodeURL(nodeURL)
			if err != nil {
				return nil, err
			}
			return gce.Get(project, zone, instance).Do()
		},
		annotators: []annotator{
			{
				name: "instance-id-reconciler",
				annotate: func(node *core.Node, instance *compute.Instance) bool {
					eid := strconv.FormatUint(instance.Id, 10)
					if len(node.ObjectMeta.Annotations) != 0 && eid == node.ObjectMeta.Annotations[InstanceIDAnnotationKey] {
						// node restarted but no update of ExternalID required
						return false
					}
					if node.ObjectMeta.Annotations == nil {
						node.ObjectMeta.Annotations = make(map[string]string)
					}
					node.ObjectMeta.Annotations[InstanceIDAnnotationKey] = eid
					return true
				},
			},
			{
				name: "labels-reconciler",
				annotate: func(node *core.Node, instance *compute.Instance) bool {
					labels, err := extractKubeLabels(instance)
					if err != nil {
						if err != errNoMetadata {
							klog.Errorf("Error reconciling labels: %v", err)
						}
						return false
					}

					if node.ObjectMeta.Labels == nil {
						node.ObjectMeta.Labels = make(map[string]string)
					}

					for _, key := range ownedKubeLabels {
						delete(node.ObjectMeta.Labels, key)
					}

					for key, value := range labels {
						node.ObjectMeta.Labels[key] = value
					}

					return true
				},
			},
		},
	}
	nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    na.add,
		UpdateFunc: na.update,
	})
	return na, nil
}

func (na *nodeAnnotator) add(obj interface{}) {
	na.enqueue(obj)
}

func (na *nodeAnnotator) update(obj, oldObj interface{}) {
	node := obj.(*core.Node)
	oldNode := oldObj.(*core.Node)
	if node.Status.NodeInfo.BootID != oldNode.Status.NodeInfo.BootID {
		na.enqueue(obj)
	}
}

func (na *nodeAnnotator) enqueue(obj interface{}) {
	key, err := controller.KeyFunc(obj)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("couldn't get key for object %+v: %v", obj, err))
		return
	}
	na.queue.Add(key)
}

func (na *nodeAnnotator) Run(workers int, stopCh <-chan struct{}) {
	if !controller.WaitForCacheSync("node-annotator", stopCh, na.hasSynced) {
		return
	}
	for i := 0; i < workers; i++ {
		go wait.Until(na.work, time.Second, stopCh)
	}
	<-stopCh
}

func (na *nodeAnnotator) processNextWorkItem() bool {
	key, quit := na.queue.Get()
	if quit {
		return false
	}
	defer na.queue.Done(key)

	na.sync(key.(string))
	na.queue.Forget(key)

	return true
}

func (na *nodeAnnotator) work() {
	for na.processNextWorkItem() {
	}
}

func (na *nodeAnnotator) sync(key string) {
	node, err := na.ns.Get(key)
	if err != nil {
		klog.Errorf("Sync %v failed with: %v", key, err)
		if errors.IsNotFound(err) {
			klog.Infof("Node %v doesn't exist, dropping from the queue", key)
			return
		}
		na.queue.Add(key)
		return
	}

	instance, err := na.getInstance(node.Spec.ProviderID)
	if err != nil {
		klog.Errorf("Sync %v failed with: %v", key, err)
		na.queue.Add(key)
		return
	}

	var update bool
	for _, ann := range na.annotators {
		modified := ann.annotate(node, instance)
		if modified {
			klog.Infof("%q annotater acting on %q", ann.name, node.Name)
		}
		update = update || modified
	}
	if !update {
		return
	}

	if _, err := na.c.CoreV1().Nodes().Update(node); err != nil {
		klog.Errorf("Sync %v failed with: %v", key, err)
		na.queue.Add(key)
		return
	}
}

type annotator struct {
	name     string
	annotate func(*core.Node, *compute.Instance) bool
}

func parseNodeURL(nodeURL string) (project, zone, instance string, err error) {
	// We only expect to handle strings that look like:
	// gce://project/zone/instance. Splitting by "/", the parts should look
	// like: ["gce:", "", "project", "zone", "instance"]
	parts := strings.Split(nodeURL, "/")
	if len(parts) != 5 {
		return "", "", "", fmt.Errorf("failed to parse %q: expected a three part path", nodeURL)
	}
	if parts[0] != "gce:" {
		return "", "", "", fmt.Errorf("instance %q doesn't run on gce", nodeURL)
	}
	if len(parts[1]) != 0 {
		return "", "", "", fmt.Errorf("failed to parse %q: part one of path to have length 0", nodeURL)
	}
	return parts[2], parts[3], parts[4], nil
}

// TODO: move this to instance.Labels. This is gross.
func extractKubeLabels(instance *compute.Instance) (map[string]string, error) {
	const labelsKey = "kube-labels"

	if instance.Metadata == nil {
		return nil, errNoMetadata
	}

	var kubeLabels *string
	for _, item := range instance.Metadata.Items {
		if item == nil || item.Key != labelsKey {
			continue
		}
		if item.Value == nil {
			return nil, fmt.Errorf("instance %q had nil %q", instance.SelfLink, labelsKey)
		}
		kubeLabels = item.Value
	}
	if kubeLabels == nil {
		return nil, errNoMetadata
	}
	if len(*kubeLabels) == 0 {
		return make(map[string]string), nil
	}

	labels := make(map[string]string)
	for _, kv := range strings.Split(*kubeLabels, ",") {
		parts := strings.SplitN(kv, "=", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("instance %q had malformed label pair: %q", instance.SelfLink, kv)
		}
		labels[parts[0]] = parts[1]
	}
	if err := unversionedvalidation.ValidateLabels(labels, field.NewPath("labels")); len(err) != 0 {
		return nil, fmt.Errorf("instance %q had invalid label(s): %v", instance.SelfLink, err)
	}

	return labels, nil
}
<commit_msg>Trim gce:// prefix from instanceURL before split.<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	compute "google.golang.org/api/compute/v1"

	core "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	unversionedvalidation "k8s.io/apimachinery/pkg/apis/meta/v1/validation"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apimachinery/pkg/util/wait"
	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/klog"
	"k8s.io/kubernetes/pkg/controller"
)

// InstanceIDAnnotationKey is the node annotation key where the external ID is written.
const InstanceIDAnnotationKey = "container.googleapis.com/instance_id"

var errNoMetadata = fmt.Errorf("instance did not have 'kube-labels' metadata")

type nodeAnnotator struct {
	c          clientset.Interface
	ns         corelisters.NodeLister
	hasSynced  func() bool
	queue      workqueue.RateLimitingInterface
	annotators []annotator
	// for testing
	getInstance func(nodeURL string) (*compute.Instance, error)
}

func newNodeAnnotator(client clientset.Interface, nodeInformer coreinformers.NodeInformer, cs *compute.Service) (*nodeAnnotator, error) {
	gce := compute.NewInstancesService(cs)

	// TODO(mikedanese): create a registry for the labels that GKE uses. This was
	// lifted from node_startup.go and the naming scheme is adhoc and
	// inconsistent.
	ownedKubeLabels := []string{
		"cloud.google.com/gke-nodepool",
		"cloud.google.com/gke-local-ssd",
		"cloud.google.com/gke-local-scsi-ssd",
		"cloud.google.com/gke-local-nvme-ssd",
		"cloud.google.com/gke-preemptible",
		"cloud.google.com/gke-gpu",
		"cloud.google.com/gke-accelerator",
		"beta.kubernetes.io/fluentd-ds-ready",
		"beta.kubernetes.io/kube-proxy-ds-ready",
		"beta.kubernetes.io/masq-agent-ds-ready",
		"projectcalico.org/ds-ready",
		"beta.kubernetes.io/metadata-proxy-ready",
		"addon.gke.io/node-local-dns-ds-ready",
	}

	na := &nodeAnnotator{
		c:         client,
		ns:        nodeInformer.Lister(),
		hasSynced: nodeInformer.Informer().HasSynced,
		queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewMaxOfRateLimiter(
			workqueue.NewItemExponentialFailureRateLimiter(200*time.Millisecond, 1000*time.Second),
		), "node-annotator"),
		getInstance: func(nodeURL string) (*compute.Instance, error) {
			project, zone, instance, err := parseNodeURL(nodeURL)
			if err != nil {
				return nil, err
			}
			return gce.Get(project, zone, instance).Do()
		},
		annotators: []annotator{
			{
				name: "instance-id-reconciler",
				annotate: func(node *core.Node, instance *compute.Instance) bool {
					eid := strconv.FormatUint(instance.Id, 10)
					if len(node.ObjectMeta.Annotations) != 0 && eid == node.ObjectMeta.Annotations[InstanceIDAnnotationKey] {
						// node restarted but no update of ExternalID required
						return false
					}
					if node.ObjectMeta.Annotations == nil {
						node.ObjectMeta.Annotations = make(map[string]string)
					}
					node.ObjectMeta.Annotations[InstanceIDAnnotationKey] = eid
					return true
				},
			},
			{
				name: "labels-reconciler",
				annotate: func(node *core.Node, instance *compute.Instance) bool {
					labels, err := extractKubeLabels(instance)
					if err != nil {
						if err != errNoMetadata {
							klog.Errorf("Error reconciling labels: %v", err)
						}
						return false
					}

					if node.ObjectMeta.Labels == nil {
						node.ObjectMeta.Labels = make(map[string]string)
					}

					for _, key := range ownedKubeLabels {
						delete(node.ObjectMeta.Labels, key)
					}

					for key, value := range labels {
						node.ObjectMeta.Labels[key] = value
					}

					return true
				},
			},
		},
	}
	nodeInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    na.add,
		UpdateFunc: na.update,
	})
	return na, nil
}

func (na *nodeAnnotator) add(obj interface{}) {
	na.enqueue(obj)
}

func (na *nodeAnnotator) update(obj, oldObj interface{}) {
	node := obj.(*core.Node)
	oldNode := oldObj.(*core.Node)
	if node.Status.NodeInfo.BootID != oldNode.Status.NodeInfo.BootID {
		na.enqueue(obj)
	}
}

func (na *nodeAnnotator) enqueue(obj interface{}) {
	key, err := controller.KeyFunc(obj)
	if err != nil {
		utilruntime.HandleError(fmt.Errorf("couldn't get key for object %+v: %v", obj, err))
		return
	}
	na.queue.Add(key)
}

func (na *nodeAnnotator) Run(workers int, stopCh <-chan struct{}) {
	if !controller.WaitForCacheSync("node-annotator", stopCh, na.hasSynced) {
		return
	}
	for i := 0; i < workers; i++ {
		go wait.Until(na.work, time.Second, stopCh)
	}
	<-stopCh
}

func (na *nodeAnnotator) processNextWorkItem() bool {
	key, quit := na.queue.Get()
	if quit {
		return false
	}
	defer na.queue.Done(key)

	na.sync(key.(string))
	na.queue.Forget(key)

	return true
}

func (na *nodeAnnotator) work() {
	for na.processNextWorkItem() {
	}
}

func (na *nodeAnnotator) sync(key string) {
	node, err := na.ns.Get(key)
	if err != nil {
		klog.Errorf("Sync %v failed with: %v", key, err)
		if errors.IsNotFound(err) {
			klog.Infof("Node %v doesn't exist, dropping from the queue", key)
			return
		}
		na.queue.Add(key)
		return
	}

	instance, err := na.getInstance(node.Spec.ProviderID)
	if err != nil {
		klog.Errorf("Sync %v failed with: %v", key, err)
		na.queue.Add(key)
		return
	}

	var update bool
	for _, ann := range na.annotators {
		modified := ann.annotate(node, instance)
		if modified {
			klog.Infof("%q annotater acting on %q", ann.name, node.Name)
		}
		update = update || modified
	}
	if !update {
		return
	}

	if _, err := na.c.CoreV1().Nodes().Update(node); err != nil {
		klog.Errorf("Sync %v failed with: %v", key, err)
		na.queue.Add(key)
		return
	}
}

type annotator struct {
	name     string
	annotate func(*core.Node, *compute.Instance) bool
}

func parseNodeURL(nodeURL string) (project, zone, instance string, err error) {
	// We only expect to handle strings that look like:
	// gce://project/zone/instance
	if !strings.HasPrefix(nodeURL, "gce://") {
		return "", "", "", fmt.Errorf("instance %q doesn't run on gce", nodeURL)
	}
	parts := strings.Split(strings.TrimPrefix(nodeURL, "gce://"), "/")
	if len(parts) != 3 {
		return "", "", "", fmt.Errorf("failed to parse %q: expected a three part path", nodeURL)
	}
	return parts[0], parts[1], parts[2], nil
}

// TODO: move this to instance.Labels. This is gross.
func extractKubeLabels(instance *compute.Instance) (map[string]string, error) {
	const labelsKey = "kube-labels"

	if instance.Metadata == nil {
		return nil, errNoMetadata
	}

	var kubeLabels *string
	for _, item := range instance.Metadata.Items {
		if item == nil || item.Key != labelsKey {
			continue
		}
		if item.Value == nil {
			return nil, fmt.Errorf("instance %q had nil %q", instance.SelfLink, labelsKey)
		}
		kubeLabels = item.Value
	}
	if kubeLabels == nil {
		return nil, errNoMetadata
	}
	if len(*kubeLabels) == 0 {
		return make(map[string]string), nil
	}

	labels := make(map[string]string)
	for _, kv := range strings.Split(*kubeLabels, ",") {
		parts := strings.SplitN(kv, "=", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("instance %q had malformed label pair: %q", instance.SelfLink, kv)
		}
		labels[parts[0]] = parts[1]
	}
	if err := unversionedvalidation.ValidateLabels(labels, field.NewPath("labels")); len(err) != 0 {
		return nil, fmt.Errorf("instance %q had invalid label(s): %v", instance.SelfLink, err)
	}

	return labels, nil
}
<|endoftext|>


<commit_before>package genhandler

import (
	"bytes"
	"strings"
	"text/template"

	"github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	"github.com/pkg/errors"
)

var (
	errNoTargetService = errors.New("no target service defined in the file")
)

type param struct {
	*descriptor.File
	Imports    []descriptor.GoPackage
	SwagBuffer []byte
}

func applyTemplate(p param) (string, error) {
	w := bytes.NewBuffer(nil)
	if err := headerTemplate.Execute(w, p); err != nil {
		return "", err
	}

	if err := regTemplate.ExecuteTemplate(w, "base", p); err != nil {
		return "", err
	}

	type swaggerTmpl struct {
		FileName string
		Swagger  string
	}

	if err := footerTemplate.Execute(w, p); err != nil {
		return "", err
	}

	if err := patternsTemplate.ExecuteTemplate(w, "base", p); err != nil {
		return "", err
	}

	return w.String(), nil
}

var (
	funcMap = template.FuncMap{
		"dotToUnderscore": func(s string) string { return strings.Replace(strings.Replace(s, ".", "_", -1), "/", "_", -1) },
		"byteStr":         func(b []byte) string { return string(b) },
		"escapeBackTicks": func(s string) string { return strings.Replace(s, "`", "` + \"``\" + `", -1) },
	}

	headerTemplate = template.Must(template.New("header").Parse(`
// Code generated by protoc-gen-goclay
// source: {{.GetName}}
// DO NOT EDIT!

/*
Package {{.GoPkg.Name}} is a self-registering gRPC and JSON+Swagger service definition.

It conforms to the github.com/utrack/clay Service interface.
*/
package {{.GoPkg.Name}}
import (
	{{range $i := .Imports}}{{if $i.Standard}}{{$i | printf "%s\n"}}{{end}}{{end}}

	{{range $i := .Imports}}{{if not $i.Standard}}{{$i | printf "%s\n"}}{{end}}{{end}}
)

// Update your shared lib or downgrade generator to v1 if there's an error
var _ = transport.IsVersion2

var _ chi.Router
var _ runtime.Marshaler
`))
	regTemplate = template.Must(template.New("svc-reg").Funcs(funcMap).Parse(`
{{define "base"}}
{{range $svc := .Services}}
// {{$svc.GetName}}Desc is a descriptor/registrator for the {{$svc.GetName}}Server.
type {{$svc.GetName}}Desc struct {
      svc {{$svc.GetName}}Server
}

// New{{$svc.GetName}}ServiceDesc creates new registrator for the {{$svc.GetName}}Server.
func New{{$svc.GetName}}ServiceDesc(svc {{$svc.GetName}}Server) *{{$svc.GetName}}Desc {
      return &{{$svc.GetName}}Desc{svc:svc}
}

// RegisterGRPC implements service registrator interface.
func (d *{{$svc.GetName}}Desc) RegisterGRPC(s *grpc.Server) {
      Register{{$svc.GetName}}Server(s,d.svc)
}

// SwaggerDef returns this file's Swagger definition.
func (d *{{$svc.GetName}}Desc) SwaggerDef() []byte {
      return _swaggerDef_{{dotToUnderscore $.GetName}}
}

// RegisterHTTP registers this service's HTTP handlers/bindings.
func (d *{{$svc.GetName}}Desc) RegisterHTTP(mux transport.Router) {
	{{range $m := $svc.Methods}}
	// Handlers for {{$m.GetName}}
	{{range $b := $m.Bindings}}
	mux.MethodFunc(pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}},"{{$b.HTTPMethod}}", func(w http.ResponseWriter, r *http.Request) {
          defer r.Body.Close()

	  var req {{$m.RequestType.GetName}}
          err := unmarshaler_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}(r,&req)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"couldn't parse request"))
	    return
	  }

	  ret,err := d.svc.{{$m.GetName}}(r.Context(),&req)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"returned from handler"))
	    return
	  }

          _,outbound := httpruntime.MarshalerForRequest(r)
          w.Header().Set("Content-Type", outbound.ContentType())
	  err = outbound.Marshal(w, ret)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"couldn't write response"))
	    return
	  }
      })
      {{end}}
      {{end}}
}
{{end}}
{{end}} // base service handler ended
`))

	footerTemplate = template.Must(template.New("footer").Funcs(funcMap).Parse(`
var _swaggerDef_{{dotToUnderscore .GetName}} = []byte(` + "`" + `{{escapeBackTicks (byteStr .SwagBuffer)}}` + `
` + "`)" + `
`))

	patternsTemplate = template.Must(template.New("patterns").Parse(`
{{define "base"}}
var (
{{range $svc := .Services}}
{{range $m := $svc.Methods}}
{{range $b := $m.Bindings}}
	pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}} = "{{$b.PathTmpl.Template}}"
        unmarshaler_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}} = func(r *http.Request,req *{{$m.RequestType.GetName}}) error {
        {{if eq $b.HTTPMethod "GET" }}
          {{template "unget" .}}
        {{end}}
        {{if eq $b.HTTPMethod "POST" }}
          {{template "unpost" .}}
        {{end}}
        }
{{end}}
{{end}}
{{end}}
)
{{end}}
{{define "unpost"}}
          inbound,_ := httpruntime.MarshalerForRequest(r)
	  return errors.Wrap(inbound.Unmarshal(r.Body,req),"couldn't read request JSON")
{{end}}
{{define "unget"}}
	  rctx := chi.RouteContext(r.Context())
          if rctx == nil {
            panic("Only chi router is supported for GETs atm")
	  }
          for pos,k := range rctx.URLParams.Keys {
	    runtime.PopulateFieldFromPath(req, k, rctx.URLParams.Values[pos])
          }
          return nil
{{end}}
`))
)
<commit_msg>Add PUT and DELETE  support<commit_after>package genhandler

import (
	"bytes"
	"strings"
	"text/template"

	"github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway/descriptor"
	"github.com/pkg/errors"
)

var (
	errNoTargetService = errors.New("no target service defined in the file")
)

type param struct {
	*descriptor.File
	Imports    []descriptor.GoPackage
	SwagBuffer []byte
}

func applyTemplate(p param) (string, error) {
	w := bytes.NewBuffer(nil)
	if err := headerTemplate.Execute(w, p); err != nil {
		return "", err
	}

	if err := regTemplate.ExecuteTemplate(w, "base", p); err != nil {
		return "", err
	}

	type swaggerTmpl struct {
		FileName string
		Swagger  string
	}

	if err := footerTemplate.Execute(w, p); err != nil {
		return "", err
	}

	if err := patternsTemplate.ExecuteTemplate(w, "base", p); err != nil {
		return "", err
	}

	return w.String(), nil
}

var (
	funcMap = template.FuncMap{
		"dotToUnderscore": func(s string) string { return strings.Replace(strings.Replace(s, ".", "_", -1), "/", "_", -1) },
		"byteStr":         func(b []byte) string { return string(b) },
		"escapeBackTicks": func(s string) string { return strings.Replace(s, "`", "` + \"``\" + `", -1) },
	}

	headerTemplate = template.Must(template.New("header").Parse(`
// Code generated by protoc-gen-goclay
// source: {{.GetName}}
// DO NOT EDIT!

/*
Package {{.GoPkg.Name}} is a self-registering gRPC and JSON+Swagger service definition.

It conforms to the github.com/utrack/clay Service interface.
*/
package {{.GoPkg.Name}}
import (
	{{range $i := .Imports}}{{if $i.Standard}}{{$i | printf "%s\n"}}{{end}}{{end}}

	{{range $i := .Imports}}{{if not $i.Standard}}{{$i | printf "%s\n"}}{{end}}{{end}}
)

// Update your shared lib or downgrade generator to v1 if there's an error
var _ = transport.IsVersion2

var _ chi.Router
var _ runtime.Marshaler
`))
	regTemplate = template.Must(template.New("svc-reg").Funcs(funcMap).Parse(`
{{define "base"}}
{{range $svc := .Services}}
// {{$svc.GetName}}Desc is a descriptor/registrator for the {{$svc.GetName}}Server.
type {{$svc.GetName}}Desc struct {
      svc {{$svc.GetName}}Server
}

// New{{$svc.GetName}}ServiceDesc creates new registrator for the {{$svc.GetName}}Server.
func New{{$svc.GetName}}ServiceDesc(svc {{$svc.GetName}}Server) *{{$svc.GetName}}Desc {
      return &{{$svc.GetName}}Desc{svc:svc}
}

// RegisterGRPC implements service registrator interface.
func (d *{{$svc.GetName}}Desc) RegisterGRPC(s *grpc.Server) {
      Register{{$svc.GetName}}Server(s,d.svc)
}

// SwaggerDef returns this file's Swagger definition.
func (d *{{$svc.GetName}}Desc) SwaggerDef() []byte {
      return _swaggerDef_{{dotToUnderscore $.GetName}}
}

// RegisterHTTP registers this service's HTTP handlers/bindings.
func (d *{{$svc.GetName}}Desc) RegisterHTTP(mux transport.Router) {
	{{range $m := $svc.Methods}}
	// Handlers for {{$m.GetName}}
	{{range $b := $m.Bindings}}
	mux.MethodFunc(pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}},"{{$b.HTTPMethod}}", func(w http.ResponseWriter, r *http.Request) {
          defer r.Body.Close()

	  var req {{$m.RequestType.GetName}}
          err := unmarshaler_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}}(r,&req)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"couldn't parse request"))
	    return
	  }

	  ret,err := d.svc.{{$m.GetName}}(r.Context(),&req)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"returned from handler"))
	    return
	  }

          _,outbound := httpruntime.MarshalerForRequest(r)
          w.Header().Set("Content-Type", outbound.ContentType())
	  err = outbound.Marshal(w, ret)
	  if err != nil {
	    httpruntime.SetError(r.Context(),r,w,errors.Wrap(err,"couldn't write response"))
	    return
	  }
      })
      {{end}}
      {{end}}
}
{{end}}
{{end}} // base service handler ended
`))

	footerTemplate = template.Must(template.New("footer").Funcs(funcMap).Parse(`
var _swaggerDef_{{dotToUnderscore .GetName}} = []byte(` + "`" + `{{escapeBackTicks (byteStr .SwagBuffer)}}` + `
` + "`)" + `
`))

	patternsTemplate = template.Must(template.New("patterns").Parse(`
{{define "base"}}
var (
{{range $svc := .Services}}
{{range $m := $svc.Methods}}
{{range $b := $m.Bindings}}
	pattern_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}} = "{{$b.PathTmpl.Template}}"
        unmarshaler_goclay_{{$svc.GetName}}_{{$m.GetName}}_{{$b.Index}} = func(r *http.Request,req *{{$m.RequestType.GetName}}) error {
        {{if eq $b.HTTPMethod "GET" }}
          {{template "unget" .}}
        {{end}}
        {{if eq $b.HTTPMethod "POST" }}
          {{template "unpost" .}}
        {{end}}
		{{if eq $b.HTTPMethod "PUT" }}
          {{template "unpost" .}}
        {{end}}
		{{if eq $b.HTTPMethod "DELETE" }}
          {{template "unpost" .}}
        {{end}}
        }
{{end}}
{{end}}
{{end}}
)
{{end}}
{{define "unpost"}}
	  rctx := chi.RouteContext(r.Context())
          if rctx == nil {
            panic("Only chi router is supported")
	  }
          for pos,k := range rctx.URLParams.Keys {
	    runtime.PopulateFieldFromPath(req, k, rctx.URLParams.Values[pos])
          }

          inbound,_ := httpruntime.MarshalerForRequest(r)
	  return errors.Wrap(inbound.Unmarshal(r.Body,req),"couldn't read request JSON")
{{end}}
{{define "unget"}}
	  rctx := chi.RouteContext(r.Context())
          if rctx == nil {
            panic("Only chi router is supported for GETs atm")
	  }
          for pos,k := range rctx.URLParams.Keys {
	    runtime.PopulateFieldFromPath(req, k, rctx.URLParams.Values[pos])
          }
          return nil
{{end}}
`))
)
<|endoftext|>


<commit_before>package godo

// Version is the current version
var Version = "2.0.0-pre"
<commit_msg>sync up VERSION.go file with tag<commit_after>package godo

// Version is the current version
var Version = "1.1.0"
<|endoftext|>


<commit_before>package darwin

import (
	"database/sql/driver"
	"strings"
	"testing"
	"time"

	"gopkg.in/DATA-DOG/go-sqlmock.v1"
)

func TestMigrate(t *testing.T) {
	db, mock, err := sqlmock.New()

	if err != nil {
		t.Errorf("sqlmock.New().error != nil, wants nil")
	}

	defer db.Close()

	dialect := MySQLDialect{}

	mock.ExpectExec(escapeQuery(dialect.CreateTableSQL())).WillReturnResult(sqlmock.NewResult(0, 0))

	query := "CREATE TABLE people (id INT AUTO_INCREMENT NOT NULL, PRIMARY KEY (id));"
	migration := Migration{
		Version:     1.0,
		Description: "Creating table people",
		Script:      strings.NewReader(query),
	}

	mock.ExpectExec(escapeQuery(query)).WillReturnResult(sqlmock.NewResult(1, 1))

	migrations := []Migration{migration}

	mock.ExpectExec(escapeQuery(dialect.MigrateSQL())).WithArgs(
		1.0, "Creating table people", "7ebca1c6f05333a728a8db4629e8d543",
		anyRFC3339{}, sqlmock.AnyArg(), true).WillReturnResult(sqlmock.NewResult(1, 1))

	Migrate(db, dialect, migrations)

	if err := mock.ExpectationsWereMet(); err != nil {
		t.Errorf("there were unfulfilled expections: %s", err)
	}
}

func escapeQuery(s string) string {
	s1 := strings.Replace(s, ")", "\\)", -1)
	s1 = strings.Replace(s1, "(", "\\(", -1)
	s1 = strings.Replace(s1, "?", "\\?", -1)
	return s1
}

type anyRFC3339 struct{}

// Match satisfies sqlmock.Argument interface
func (a anyRFC3339) Match(v driver.Value) bool {
	_, ok := v.(string)

	if !ok {
		return false
	}

	_, err := time.Parse(time.RFC3339, v.(string))

	if err != nil {
		return false
	}

	return true
}
<commit_msg>Better escapeQuery func<commit_after>package darwin

import (
	"database/sql/driver"
	"regexp"
	"strings"
	"testing"
	"time"

	"gopkg.in/DATA-DOG/go-sqlmock.v1"
)

func TestMigrate(t *testing.T) {
	db, mock, err := sqlmock.New()

	if err != nil {
		t.Errorf("sqlmock.New().error != nil, wants nil")
	}

	defer db.Close()

	dialect := MySQLDialect{}

	mock.ExpectExec(escapeQuery(dialect.CreateTableSQL())).WillReturnResult(sqlmock.NewResult(0, 0))

	query := "CREATE TABLE people (id INT AUTO_INCREMENT NOT NULL, PRIMARY KEY (id));"
	migration := Migration{
		Version:     1.0,
		Description: "Creating table people",
		Script:      strings.NewReader(query),
	}

	mock.ExpectExec(escapeQuery(query)).WillReturnResult(sqlmock.NewResult(1, 1))

	migrations := []Migration{migration}

	mock.ExpectExec(escapeQuery(dialect.MigrateSQL())).WithArgs(
		1.0, "Creating table people", "7ebca1c6f05333a728a8db4629e8d543",
		anyRFC3339{}, sqlmock.AnyArg(), true).WillReturnResult(sqlmock.NewResult(1, 1))

	Migrate(db, dialect, migrations)

	if err := mock.ExpectationsWereMet(); err != nil {
		t.Errorf("there were unfulfilled expections: %s", err)
	}
}

func escapeQuery(s string) string {
	s1 := strings.NewReplacer(
		")", "\\)",
		"(", "\\(",
		"?", "\\?",
		"\n", " ",
		"\r", " ",
		"\t", " ",
	).Replace(s)

	re := regexp.MustCompile("\\s+")
	s1 = strings.TrimSpace(re.ReplaceAllString(s1, " "))
	return s1
}

type anyRFC3339 struct{}

// Match satisfies sqlmock.Argument interface
func (a anyRFC3339) Match(v driver.Value) bool {
	_, ok := v.(string)

	if !ok {
		return false
	}

	_, err := time.Parse(time.RFC3339, v.(string))

	if err != nil {
		return false
	}

	return true
}
<|endoftext|>