<commit_before>// Copyright 2017 The Kubernetes Dashboard Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package handler

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"

	restful "github.com/emicklei/go-restful"
	"gopkg.in/igm/sockjs-go.v2/sockjs"
	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/remotecommand"
)

// PtyHandler is what remotecommand expects from a pty
type PtyHandler interface {
	io.Reader
	io.Writer
	remotecommand.TerminalSizeQueue
}

// TerminalSession implements PtyHandler (using a SockJS connection)
type TerminalSession struct {
	id            string
	bound         chan error
	sockJSSession sockjs.Session
	sizeChan      chan remotecommand.TerminalSize
}

// TerminalMessage is the messaging protocol between ShellController and TerminalSession.
//
// OP      DIRECTION  FIELD(S) USED  DESCRIPTION
// ---------------------------------------------------------------------
// bind    fe->be     SessionID      Id sent back from TerminalReponse
// stdin   fe->be     Data           Keystrokes/paste buffer
// resize  fe->be     Rows, Cols     New terminal size
// stdout  be->fe     Data           Output from the process
// toast   be->fe     Data           OOB message to be shown to the user
type TerminalMessage struct {
	Op, Data, SessionID string
	Rows, Cols          uint16
}

// TerminalSize handles pty->process resize events
// Called in a loop from remotecommand as long as the process is running
func (t TerminalSession) Next() *remotecommand.TerminalSize {
	select {
	case size := <-t.sizeChan:
		return &size
	}
}

// Read handles pty->process messages (stdin, resize)
// Called in a loop from remotecommand as long as the process is running
func (t TerminalSession) Read(p []byte) (int, error) {
	m, err := t.sockJSSession.Recv()
	if err != nil {
		return 0, err
	}

	var msg TerminalMessage
	if err := json.Unmarshal([]byte(m), &msg); err != nil {
		return 0, err
	}

	switch msg.Op {
	case "stdin":
		return copy(p, msg.Data), nil
	case "resize":
		t.sizeChan <- remotecommand.TerminalSize{msg.Cols, msg.Rows}
		return 0, nil
	default:
		return 0, fmt.Errorf("unknown message type '%s'", msg.Op)
	}
}

// Write handles process->pty stdout
// Called from remotecommand whenever there is any output
func (t TerminalSession) Write(p []byte) (int, error) {
	msg, err := json.Marshal(TerminalMessage{
		Op:   "stdout",
		Data: string(p),
	})
	if err != nil {
		return 0, err
	}

	if err = t.sockJSSession.Send(string(msg)); err != nil {
		return 0, err
	}
	return len(p), nil
}

// Toast can be used to send the user any OOB messages
// hterm puts these in the center of the terminal
func (t TerminalSession) Toast(p string) error {
	msg, err := json.Marshal(TerminalMessage{
		Op:   "toast",
		Data: p,
	})
	if err != nil {
		return err
	}

	if err = t.sockJSSession.Send(string(msg)); err != nil {
		return err
	}
	return nil
}

// Close shuts down the SockJS connection and sends the status code and reason to the client
// Can happen if the process exits or if there is an error starting up the process
// For now the status code is unused and reason is shown to the user (unless "")
func (t TerminalSession) Close(status uint32, reason string) {
	t.sockJSSession.Close(status, reason)
}

// terminalSessions stores a map of all TerminalSession objects
// FIXME: this structure needs locking
var terminalSessions = make(map[string]TerminalSession)

// handleTerminalSession is Called by net/http for any new /api/sockjs connections
func handleTerminalSession(session sockjs.Session) {
	var (
		buf             string
		err             error
		msg             TerminalMessage
		terminalSession TerminalSession
		ok              bool
	)

	if buf, err = session.Recv(); err != nil {
		log.Printf("handleTerminalSession: can't Recv: %v", err)
		return
	}

	if err = json.Unmarshal([]byte(buf), &msg); err != nil {
		log.Printf("handleTerminalSession: can't UnMarshal (%v): %s", err, buf)
		return
	}

	if msg.Op != "bind" {
		log.Printf("handleTerminalSession: expected 'bind' message, got: %s", buf)
		return
	}

	if terminalSession, ok = terminalSessions[msg.SessionID]; !ok {
		log.Printf("handleTerminalSession: can't find session '%s'", msg.SessionID)
		return
	}

	terminalSession.sockJSSession = session
	terminalSession.bound <- nil
	terminalSessions[msg.SessionID] = terminalSession
}

// CreateAttachHandler is called from main for /api/sockjs
func CreateAttachHandler(path string) http.Handler {
	return sockjs.NewHandler(path, sockjs.DefaultOptions, handleTerminalSession)
}

// startProcess is called by handleAttach
// Executed cmd in the container specified in request and connects it up with the ptyHandler (a session)
func startProcess(k8sClient kubernetes.Interface, cfg *rest.Config, request *restful.Request, cmd []string, ptyHandler PtyHandler) error {
	namespace := request.PathParameter("namespace")
	podName := request.PathParameter("pod")
	containerName := request.PathParameter("container")

	req := k8sClient.Core().RESTClient().Post().
		Resource("pods").
		Name(podName).
		Namespace(namespace).
		SubResource("exec")

	req.VersionedParams(&api.PodExecOptions{
		Container: containerName,
		Command:   cmd,
		Stdin:     true,
		Stdout:    true,
		Stderr:    true,
		TTY:       true,
	}, api.ParameterCodec)

	exec, err := remotecommand.NewExecutor(cfg, "POST", req.URL())
	if err != nil {
		return err
	}

	err = exec.Stream(remotecommand.StreamOptions{
		SupportedProtocols: remotecommandconsts.SupportedStreamingProtocols,
		Stdin:              ptyHandler,
		Stdout:             ptyHandler,
		Stderr:             ptyHandler,
		TerminalSizeQueue:  ptyHandler,
		Tty:                true,
	})
	if err != nil {
		return err
	}

	return nil
}

// genTerminalSessionId generates a random session ID string. The format is not really interesting.
// This ID is used to identify the session when the client opens the SockJS connection.
// Not the same as the SockJS session id! We can't use that as that is generated
// on the client side and we don't have it yet at this point.
func genTerminalSessionId() (string, error) {
	bytes := make([]byte, 16)
	if _, err := rand.Read(bytes); err != nil {
		return "", err
	}
	id := make([]byte, hex.EncodedLen(len(bytes)))
	hex.Encode(id, bytes)
	return string(id), nil
}

// isValidShell checks if the shell is an allowed one
func isValidShell(validShells []string, shell string) bool {
	for _, validShell := range validShells {
		if validShell == shell {
			return true
		}
	}
	return false
}

// WaitForTerminal is called from apihandler.handleAttach as a goroutine
// Waits for the SockJS connection to be opened by the client the session to be bound in handleTerminalSession
func WaitForTerminal(k8sClient kubernetes.Interface, cfg *rest.Config, request *restful.Request, sessionId string) {
	shell := request.QueryParameter("shell")

	select {
	case <-terminalSessions[sessionId].bound:
		close(terminalSessions[sessionId].bound)

		var err error
		validShells := []string{"bash", "sh"}

		if isValidShell(validShells, shell) {
			cmd := []string{shell}
			err = startProcess(k8sClient, cfg, request, cmd, terminalSessions[sessionId])
		} else {
			// No shell given or it was not valid: try some shells until one succeeds or all fail
			// FIXME: if the first shell fails then the first keyboard event is lost
			for _, testShell := range validShells {
				cmd := []string{testShell}
				if err = startProcess(k8sClient, cfg, request, cmd, terminalSessions[sessionId]); err == nil {
					break
				}
			}
		}

		if err != nil {
			terminalSessions[sessionId].Close(2, err.Error())
			return
		}

		terminalSessions[sessionId].Close(1, "Process exited")
	}
}
<commit_msg>Fix web terminal after upgrading to client-go v4 (#2407) (#2422)<commit_after>// Copyright 2017 The Kubernetes Dashboard Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package handler

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"

	restful "github.com/emicklei/go-restful"
	"gopkg.in/igm/sockjs-go.v2/sockjs"
	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/remotecommand"
)

// PtyHandler is what remotecommand expects from a pty
type PtyHandler interface {
	io.Reader
	io.Writer
	remotecommand.TerminalSizeQueue
}

// TerminalSession implements PtyHandler (using a SockJS connection)
type TerminalSession struct {
	id            string
	bound         chan error
	sockJSSession sockjs.Session
	sizeChan      chan remotecommand.TerminalSize
}

// TerminalMessage is the messaging protocol between ShellController and TerminalSession.
//
// OP      DIRECTION  FIELD(S) USED  DESCRIPTION
// ---------------------------------------------------------------------
// bind    fe->be     SessionID      Id sent back from TerminalReponse
// stdin   fe->be     Data           Keystrokes/paste buffer
// resize  fe->be     Rows, Cols     New terminal size
// stdout  be->fe     Data           Output from the process
// toast   be->fe     Data           OOB message to be shown to the user
type TerminalMessage struct {
	Op, Data, SessionID string
	Rows, Cols          uint16
}

// TerminalSize handles pty->process resize events
// Called in a loop from remotecommand as long as the process is running
func (t TerminalSession) Next() *remotecommand.TerminalSize {
	select {
	case size := <-t.sizeChan:
		return &size
	}
}

// Read handles pty->process messages (stdin, resize)
// Called in a loop from remotecommand as long as the process is running
func (t TerminalSession) Read(p []byte) (int, error) {
	m, err := t.sockJSSession.Recv()
	if err != nil {
		return 0, err
	}

	var msg TerminalMessage
	if err := json.Unmarshal([]byte(m), &msg); err != nil {
		return 0, err
	}

	switch msg.Op {
	case "stdin":
		return copy(p, msg.Data), nil
	case "resize":
		t.sizeChan <- remotecommand.TerminalSize{msg.Cols, msg.Rows}
		return 0, nil
	default:
		return 0, fmt.Errorf("unknown message type '%s'", msg.Op)
	}
}

// Write handles process->pty stdout
// Called from remotecommand whenever there is any output
func (t TerminalSession) Write(p []byte) (int, error) {
	msg, err := json.Marshal(TerminalMessage{
		Op:   "stdout",
		Data: string(p),
	})
	if err != nil {
		return 0, err
	}

	if err = t.sockJSSession.Send(string(msg)); err != nil {
		return 0, err
	}
	return len(p), nil
}

// Toast can be used to send the user any OOB messages
// hterm puts these in the center of the terminal
func (t TerminalSession) Toast(p string) error {
	msg, err := json.Marshal(TerminalMessage{
		Op:   "toast",
		Data: p,
	})
	if err != nil {
		return err
	}

	if err = t.sockJSSession.Send(string(msg)); err != nil {
		return err
	}
	return nil
}

// Close shuts down the SockJS connection and sends the status code and reason to the client
// Can happen if the process exits or if there is an error starting up the process
// For now the status code is unused and reason is shown to the user (unless "")
func (t TerminalSession) Close(status uint32, reason string) {
	t.sockJSSession.Close(status, reason)
}

// terminalSessions stores a map of all TerminalSession objects
// FIXME: this structure needs locking
var terminalSessions = make(map[string]TerminalSession)

// handleTerminalSession is Called by net/http for any new /api/sockjs connections
func handleTerminalSession(session sockjs.Session) {
	var (
		buf             string
		err             error
		msg             TerminalMessage
		terminalSession TerminalSession
		ok              bool
	)

	if buf, err = session.Recv(); err != nil {
		log.Printf("handleTerminalSession: can't Recv: %v", err)
		return
	}

	if err = json.Unmarshal([]byte(buf), &msg); err != nil {
		log.Printf("handleTerminalSession: can't UnMarshal (%v): %s", err, buf)
		return
	}

	if msg.Op != "bind" {
		log.Printf("handleTerminalSession: expected 'bind' message, got: %s", buf)
		return
	}

	if terminalSession, ok = terminalSessions[msg.SessionID]; !ok {
		log.Printf("handleTerminalSession: can't find session '%s'", msg.SessionID)
		return
	}

	terminalSession.sockJSSession = session
	terminalSession.bound <- nil
	terminalSessions[msg.SessionID] = terminalSession
}

// CreateAttachHandler is called from main for /api/sockjs
func CreateAttachHandler(path string) http.Handler {
	return sockjs.NewHandler(path, sockjs.DefaultOptions, handleTerminalSession)
}

// startProcess is called by handleAttach
// Executed cmd in the container specified in request and connects it up with the ptyHandler (a session)
func startProcess(k8sClient kubernetes.Interface, cfg *rest.Config, request *restful.Request, cmd []string, ptyHandler PtyHandler) error {
	namespace := request.PathParameter("namespace")
	podName := request.PathParameter("pod")
	containerName := request.PathParameter("container")

	req := k8sClient.Core().RESTClient().Post().
		Resource("pods").
		Name(podName).
		Namespace(namespace).
		SubResource("exec")

	req.VersionedParams(&v1.PodExecOptions{
		Container: containerName,
		Command:   cmd,
		Stdin:     true,
		Stdout:    true,
		Stderr:    true,
		TTY:       true,
	}, scheme.ParameterCodec)

	exec, err := remotecommand.NewExecutor(cfg, "POST", req.URL())
	if err != nil {
		return err
	}

	err = exec.Stream(remotecommand.StreamOptions{
		SupportedProtocols: remotecommandconsts.SupportedStreamingProtocols,
		Stdin:              ptyHandler,
		Stdout:             ptyHandler,
		Stderr:             ptyHandler,
		TerminalSizeQueue:  ptyHandler,
		Tty:                true,
	})
	if err != nil {
		return err
	}

	return nil
}

// genTerminalSessionId generates a random session ID string. The format is not really interesting.
// This ID is used to identify the session when the client opens the SockJS connection.
// Not the same as the SockJS session id! We can't use that as that is generated
// on the client side and we don't have it yet at this point.
func genTerminalSessionId() (string, error) {
	bytes := make([]byte, 16)
	if _, err := rand.Read(bytes); err != nil {
		return "", err
	}
	id := make([]byte, hex.EncodedLen(len(bytes)))
	hex.Encode(id, bytes)
	return string(id), nil
}

// isValidShell checks if the shell is an allowed one
func isValidShell(validShells []string, shell string) bool {
	for _, validShell := range validShells {
		if validShell == shell {
			return true
		}
	}
	return false
}

// WaitForTerminal is called from apihandler.handleAttach as a goroutine
// Waits for the SockJS connection to be opened by the client the session to be bound in handleTerminalSession
func WaitForTerminal(k8sClient kubernetes.Interface, cfg *rest.Config, request *restful.Request, sessionId string) {
	shell := request.QueryParameter("shell")

	select {
	case <-terminalSessions[sessionId].bound:
		close(terminalSessions[sessionId].bound)

		var err error
		validShells := []string{"bash", "sh"}

		if isValidShell(validShells, shell) {
			cmd := []string{shell}
			err = startProcess(k8sClient, cfg, request, cmd, terminalSessions[sessionId])
		} else {
			// No shell given or it was not valid: try some shells until one succeeds or all fail
			// FIXME: if the first shell fails then the first keyboard event is lost
			for _, testShell := range validShells {
				cmd := []string{testShell}
				if err = startProcess(k8sClient, cfg, request, cmd, terminalSessions[sessionId]); err == nil {
					break
				}
			}
		}

		if err != nil {
			terminalSessions[sessionId].Close(2, err.Error())
			return
		}

		terminalSessions[sessionId].Close(1, "Process exited")
	}
}
<|endoftext|>


<commit_before>package handlers

import (
	"database/sql"
	"fmt"
	"net/http"

	_ "github.com/lib/pq"
	"github.com/sporttech/urest"
)

type (
	WithTxHandler struct {
		http.Handler
	}

	dbtx struct {
		db *sql.DB
		tx *sql.Tx
	}
)

const (
	driverName = "postgres"
	dbUrl      = "postgres://"
)

var (
	dbtxs = map[*http.Request]*dbtx{}
)

func (h WithTxHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	db, err := sql.Open(driverName, dbUrl)
	if err != nil {
		http.Error(w, fmt.Sprintf("Failed to establish database connection: %v", err), http.StatusInternalServerError)
		return
	}
	defer db.Close()

	dbtxs[r] = &dbtx{db, nil}
	defer delete(dbtxs, r)

	defer func() {
		if tx := dbtxs[r].tx; tx != nil {
			tx.Rollback()
		}
	}()

	tw := &TransparentResponseWriter{w, 0, 0}
	h.Handler.ServeHTTP(tw, r)

	if tw.Success() && !urest.IsSafeRequest(r) {
		if tx := dbtxs[r].tx; tx != nil {
			tx.Commit()
		}
	}
}

func Tx(r *http.Request) *sql.Tx {
	c := dbtxs[r]
	if c.tx == nil {
		if tx, err := c.db.Begin(); err != nil {
			panic(err)
		} else {
			c.tx = tx
		}
	}
	return c.tx
}
<commit_msg>handlers/pgtx: fix races by protecting access to the dbtxs map<commit_after>package handlers

import (
	"database/sql"
	"fmt"
	"net/http"
	"sync"

	_ "github.com/lib/pq"
	"github.com/sporttech/urest"
)

type (
	WithTxHandler struct {
		http.Handler
	}

	dbtx struct {
		db *sql.DB
		tx *sql.Tx
	}
)

const (
	driverName = "postgres"
	dbUrl      = "postgres://"
)

var (
	dbtxs      = map[*http.Request]*dbtx{}
	dbtxsMutex sync.Mutex
)

func (h WithTxHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	db, err := sql.Open(driverName, dbUrl)
	if err != nil {
		http.Error(w, fmt.Sprintf("Failed to establish database connection: %v", err), http.StatusInternalServerError)
		return
	}
	defer db.Close()

	c := &dbtx{db, nil}

	dbtxsMutex.Lock()
	dbtxs[r] = c
	dbtxsMutex.Unlock()

	tw := &TransparentResponseWriter{w, 0, 0}
	h.Handler.ServeHTTP(tw, r)

	if c.tx != nil {
		if tw.Success() && !urest.IsSafeRequest(r) {
			c.tx.Commit()
		} else {
			c.tx.Rollback()
		}
	}

	dbtxsMutex.Lock()
	delete(dbtxs, r)
	dbtxsMutex.Unlock()
}

func Tx(r *http.Request) *sql.Tx {
	dbtxsMutex.Lock()
	c := dbtxs[r]
	dbtxsMutex.Unlock()

	if c.tx == nil {
		if tx, err := c.db.Begin(); err != nil {
			panic(err)
		} else {
			c.tx = tx
		}
	}
	return c.tx
}
<|endoftext|>


<commit_before>// Package tsz implements time-series compression
// it is a fork of https://github.com/dgryski/go-tsz
// which implements http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
// see devdocs/chunk-format.md for more info
package tsz

import (
	"bytes"
	"math"
	"math/bits"
)

// SeriesLong similar to Series4h, except:
// * it doesn't write t0 to the stream (for callers that track t0 corresponding to a chunk separately)
// * it doesn't store an initial delta. instead, it assumes a starting delta of 60 and uses delta-of-delta
//   encoding from the get-go.
// * it uses a more compact way to mark end-of-stream
type SeriesLong struct {
	// TODO(dgryski): timestamps in the paper are uint64
	T0  uint32 // exposed for caller convenience. do NOT set directly. set via constructor
	T   uint32 // exposed for caller convenience. do NOT set directly. may only be set via Push()
	val float64

	bw       bstream
	leading  uint8
	trailing uint8
	Finished bool // exposed for caller convenience. do NOT set directly.

	tDelta uint32
}

// New series
func NewSeriesLong(t0 uint32) *SeriesLong {
	s := SeriesLong{
		T0:      t0,
		leading: ^uint8(0),
		tDelta:  60,
	}
	return &s

}

// Bytes value of the series stream
func (s *SeriesLong) Bytes() []byte {
	return s.bw.bytes()
}

// Finish the series by writing an end-of-stream record
func (s *SeriesLong) Finish() {
	if !s.Finished {
		finishV2(&s.bw)
		s.Finished = true
	}
}

// Push a timestamp and value to the series
func (s *SeriesLong) Push(t uint32, v float64) {
	var first bool

	tDelta := t - s.T
	if s.T == 0 {
		first = true
		tDelta = t - s.T0
	}
	dod := int32(tDelta - s.tDelta)

	switch {
	case dod == 0:
		s.bw.writeBit(zero)
	case -63 <= dod && dod <= 64:
		s.bw.writeBits(0x02, 2) // '10'
		s.bw.writeBits(uint64(dod), 7)
	case -255 <= dod && dod <= 256:
		s.bw.writeBits(0x06, 3) // '110'
		s.bw.writeBits(uint64(dod), 9)
	case -2047 <= dod && dod <= 2048:
		s.bw.writeBits(0x0e, 4) // '1110'
		s.bw.writeBits(uint64(dod), 12)
	default:
		s.bw.writeBits(0x1e, 5) // '11110'
		s.bw.writeBits(uint64(dod), 32)
	}

	s.tDelta = tDelta
	s.T = t

	if first {
		// first point; write full float value
		s.bw.writeBits(math.Float64bits(v), 64)
		s.val = v
		return
	}

	vDelta := math.Float64bits(v) ^ math.Float64bits(s.val)

	if vDelta == 0 {
		s.bw.writeBit(zero)
	} else {
		s.bw.writeBit(one)

		leading := uint8(bits.LeadingZeros64(vDelta))
		trailing := uint8(bits.TrailingZeros64(vDelta))

		// clamp number of leading zeros to avoid overflow when encoding
		if leading >= 32 {
			leading = 31
		}

		// TODO(dgryski): check if it's 'cheaper' to reset the leading/trailing bits instead
		if s.leading != ^uint8(0) && leading >= s.leading && trailing >= s.trailing {
			s.bw.writeBit(zero)
			s.bw.writeBits(vDelta>>s.trailing, 64-int(s.leading)-int(s.trailing))
		} else {
			s.leading, s.trailing = leading, trailing

			s.bw.writeBit(one)
			s.bw.writeBits(uint64(leading), 5)

			// Note that if leading == trailing == 0, then sigbits == 64.  But that value doesn't actually fit into the 6 bits we have.
			// Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0).
			// So instead we write out a 0 and adjust it back to 64 on unpacking.
			sigbits := 64 - leading - trailing
			s.bw.writeBits(uint64(sigbits), 6)
			s.bw.writeBits(vDelta>>trailing, int(sigbits))
		}
	}

	s.val = v

}

// IterLong lets you iterate over a series.  It is not concurrency-safe.
func (s *SeriesLong) Iter() *IterLong {
	w := s.bw.clone()

	finishV2(w)
	iter, _ := bstreamIteratorLong(s.T0, w)
	return iter
}

// IterLong lets you iterate over a series.  It is not concurrency-safe.
type IterLong struct {
	T0 uint32

	t   uint32
	val float64

	br       bstream
	leading  uint8
	trailing uint8

	finished bool

	tDelta uint32
	err    error
}

func bstreamIteratorLong(t0 uint32, br *bstream) (*IterLong, error) {

	br.count = 8

	return &IterLong{
		T0:     t0,
		br:     *br,
		tDelta: 60,
	}, nil
}

// NewIteratorLong for the series
func NewIteratorLong(t0 uint32, b []byte) (*IterLong, error) {
	return bstreamIteratorLong(t0, newBReader(b))
}

func (it *IterLong) dod() (int32, bool) {
	var d byte
	for i := 0; i < 5; i++ {
		d <<= 1
		bit, err := it.br.readBit()
		if err != nil {
			it.err = err
			return 0, false
		}
		if bit == zero {
			break
		}
		d |= 1
	}

	var dod int32
	var sz uint
	switch d {
	case 0x00:
		// dod == 0
	case 0x02: // '10'
		sz = 7
	case 0x06: // '110'
		sz = 9
	case 0x0e: // '1110'
		sz = 12
	case 0x1e: // '11110'
		bits, err := it.br.readBits(32)
		if err != nil {
			it.err = err
			return 0, false
		}
		dod = int32(bits)
	case 0x1f: // '11111': end-of-stream
		it.finished = true
		return 0, false
	}

	if sz != 0 {
		bits, err := it.br.readBits(int(sz))
		if err != nil {
			it.err = err
			return 0, false
		}
		if bits > (1 << (sz - 1)) {
			// or something
			bits = bits - (1 << sz)
		}
		dod = int32(bits)
	}

	return dod, true
}

// Next iteration of the series iterator
func (it *IterLong) Next() bool {

	if it.err != nil || it.finished {
		return false
	}

	var first bool
	if it.t == 0 {
		it.t = it.T0
		first = true
	}

	// read delta-of-delta
	dod, ok := it.dod()
	if !ok {
		return false
	}

	it.tDelta += uint32(dod)
	it.t = it.t + it.tDelta

	if first {
		// first point. read the float raw
		v, err := it.br.readBits(64)
		if err != nil {
			it.err = err
			return false
		}

		it.val = math.Float64frombits(v)
		return true
	}

	// read compressed value
	bit, err := it.br.readBit()
	if err != nil {
		it.err = err
		return false
	}

	if bit == zero {
		// it.val = it.val
	} else {
		bit, itErr := it.br.readBit()
		if itErr != nil {
			it.err = err
			return false
		}
		if bit == zero {
			// reuse leading/trailing zero bits
			// it.leading, it.trailing = it.leading, it.trailing
		} else {
			bits, err := it.br.readBits(5)
			if err != nil {
				it.err = err
				return false
			}
			it.leading = uint8(bits)

			bits, err = it.br.readBits(6)
			if err != nil {
				it.err = err
				return false
			}
			mbits := uint8(bits)
			// 0 significant bits here means we overflowed and we actually need 64; see comment in encoder
			if mbits == 0 {
				mbits = 64
			}
			it.trailing = 64 - it.leading - mbits
		}

		mbits := int(64 - it.leading - it.trailing)
		bits, err := it.br.readBits(mbits)
		if err != nil {
			it.err = err
			return false
		}
		vbits := math.Float64bits(it.val)
		vbits ^= (bits << it.trailing)
		it.val = math.Float64frombits(vbits)
	}

	return true
}

// Values at the current iterator position
func (it *IterLong) Values() (uint32, float64) {
	return it.t, it.val
}

// Err error at the current iterator position
func (it *IterLong) Err() error {
	return it.err
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (s *SeriesLong) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)
	em := &errMarshal{w: buf}
	em.write(s.T0)
	em.write(s.leading)
	em.write(s.T)
	em.write(s.tDelta)
	em.write(s.trailing)
	em.write(s.val)
	bStream, err := s.bw.MarshalBinary()
	if err != nil {
		return nil, err
	}
	em.write(bStream)
	if em.err != nil {
		return nil, em.err
	}
	return buf.Bytes(), nil
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (s *SeriesLong) UnmarshalBinary(b []byte) error {
	buf := bytes.NewReader(b)
	em := &errMarshal{r: buf}
	em.read(&s.T0)
	em.read(&s.leading)
	em.read(&s.T)
	em.read(&s.tDelta)
	em.read(&s.trailing)
	em.read(&s.val)
	outBuf := make([]byte, buf.Len())
	em.read(outBuf)
	err := s.bw.UnmarshalBinary(outBuf)
	if err != nil {
		return err
	}
	if em.err != nil {
		return em.err
	}
	return nil
}
<commit_msg>Reset significant bits if smaller<commit_after>// Package tsz implements time-series compression
// it is a fork of https://github.com/dgryski/go-tsz
// which implements http://www.vldb.org/pvldb/vol8/p1816-teller.pdf
// see devdocs/chunk-format.md for more info
package tsz

import (
	"bytes"
	"math"
	"math/bits"
)

// SeriesLong similar to Series4h, except:
// * it doesn't write t0 to the stream (for callers that track t0 corresponding to a chunk separately)
// * it doesn't store an initial delta. instead, it assumes a starting delta of 60 and uses delta-of-delta
//   encoding from the get-go.
// * it uses a more compact way to mark end-of-stream
type SeriesLong struct {
	// TODO(dgryski): timestamps in the paper are uint64
	T0  uint32 // exposed for caller convenience. do NOT set directly. set via constructor
	T   uint32 // exposed for caller convenience. do NOT set directly. may only be set via Push()
	val float64

	bw       bstream
	leading  uint8
	trailing uint8
	Finished bool // exposed for caller convenience. do NOT set directly.

	tDelta uint32
}

// New series
func NewSeriesLong(t0 uint32) *SeriesLong {
	s := SeriesLong{
		T0:      t0,
		leading: ^uint8(0),
		tDelta:  60,
	}
	return &s

}

// Bytes value of the series stream
func (s *SeriesLong) Bytes() []byte {
	return s.bw.bytes()
}

// Finish the series by writing an end-of-stream record
func (s *SeriesLong) Finish() {
	if !s.Finished {
		finishV2(&s.bw)
		s.Finished = true
	}
}

// Push a timestamp and value to the series
func (s *SeriesLong) Push(t uint32, v float64) {
	var first bool

	tDelta := t - s.T
	if s.T == 0 {
		first = true
		tDelta = t - s.T0
	}
	dod := int32(tDelta - s.tDelta)

	switch {
	case dod == 0:
		s.bw.writeBit(zero)
	case -63 <= dod && dod <= 64:
		s.bw.writeBits(0x02, 2) // '10'
		s.bw.writeBits(uint64(dod), 7)
	case -255 <= dod && dod <= 256:
		s.bw.writeBits(0x06, 3) // '110'
		s.bw.writeBits(uint64(dod), 9)
	case -2047 <= dod && dod <= 2048:
		s.bw.writeBits(0x0e, 4) // '1110'
		s.bw.writeBits(uint64(dod), 12)
	default:
		s.bw.writeBits(0x1e, 5) // '11110'
		s.bw.writeBits(uint64(dod), 32)
	}

	s.tDelta = tDelta
	s.T = t

	if first {
		// first point; write full float value
		s.bw.writeBits(math.Float64bits(v), 64)
		s.val = v
		return
	}

	vDelta := math.Float64bits(v) ^ math.Float64bits(s.val)

	if vDelta == 0 {
		s.bw.writeBit(zero)
	} else {
		s.bw.writeBit(one)

		leading := uint8(bits.LeadingZeros64(vDelta))
		trailing := uint8(bits.TrailingZeros64(vDelta))

		// clamp number of leading zeros to avoid overflow when encoding
		if leading >= 32 {
			leading = 31
		}

		sigbits := 64 - leading - trailing
		currbits := 64 - s.leading - s.trailing

		// We have 2 cases here:
		// 1. We have at least as many leading and trailing zeroes than we previously did
		// 2. We have fewer leding or trailing zeroes that previously
		// In the case of number 2, we need to re-encode the leading/trailing zeroes count to be able to handle the number
		// of significant bits we need to write.
		// In the case of number 1, we have more options. If we have significantly more leading/trailing zeroes, then we
		// might be wasting a lot of bits by constantly writing more significant bits than we need. In this case we *could*
		// choose to "reset" by writing out the count of significant bits. It costs 11 bits to reset, so ideally we would only
		// reset when we know we will waste more bits than it would cost to reset.
		if s.leading != ^uint8(0) && leading >= s.leading && trailing >= s.trailing && currbits-sigbits < 11 {
			// Case 1: Keep the previous leading/trailing zeroes
			s.bw.writeBit(zero)
			s.bw.writeBits(vDelta>>s.trailing, int(currbits))
		} else {
			// Case 1: Keep the previous leading/trailing zeroes
			s.leading, s.trailing = leading, trailing

			s.bw.writeBit(one)
			s.bw.writeBits(uint64(leading), 5)

			// Note that if leading == trailing == 0, then sigbits == 64.  But that value doesn't actually fit into the 6 bits we have.
			// Luckily, we never need to encode 0 significant bits, since that would put us in the other case (vdelta == 0).
			// So instead we write out a 0 and adjust it back to 64 on unpacking.
			s.bw.writeBits(uint64(sigbits), 6)
			s.bw.writeBits(vDelta>>trailing, int(sigbits))
		}
	}

	s.val = v

}

// IterLong lets you iterate over a series.  It is not concurrency-safe.
func (s *SeriesLong) Iter() *IterLong {
	w := s.bw.clone()

	finishV2(w)
	iter, _ := bstreamIteratorLong(s.T0, w)
	return iter
}

// IterLong lets you iterate over a series.  It is not concurrency-safe.
type IterLong struct {
	T0 uint32

	t   uint32
	val float64

	br       bstream
	leading  uint8
	trailing uint8

	finished bool

	tDelta uint32
	err    error
}

func bstreamIteratorLong(t0 uint32, br *bstream) (*IterLong, error) {

	br.count = 8

	return &IterLong{
		T0:     t0,
		br:     *br,
		tDelta: 60,
	}, nil
}

// NewIteratorLong for the series
func NewIteratorLong(t0 uint32, b []byte) (*IterLong, error) {
	return bstreamIteratorLong(t0, newBReader(b))
}

func (it *IterLong) dod() (int32, bool) {
	var d byte
	for i := 0; i < 5; i++ {
		d <<= 1
		bit, err := it.br.readBit()
		if err != nil {
			it.err = err
			return 0, false
		}
		if bit == zero {
			break
		}
		d |= 1
	}

	var dod int32
	var sz uint
	switch d {
	case 0x00:
		// dod == 0
	case 0x02: // '10'
		sz = 7
	case 0x06: // '110'
		sz = 9
	case 0x0e: // '1110'
		sz = 12
	case 0x1e: // '11110'
		bits, err := it.br.readBits(32)
		if err != nil {
			it.err = err
			return 0, false
		}
		dod = int32(bits)
	case 0x1f: // '11111': end-of-stream
		it.finished = true
		return 0, false
	}

	if sz != 0 {
		bits, err := it.br.readBits(int(sz))
		if err != nil {
			it.err = err
			return 0, false
		}
		if bits > (1 << (sz - 1)) {
			// or something
			bits = bits - (1 << sz)
		}
		dod = int32(bits)
	}

	return dod, true
}

// Next iteration of the series iterator
func (it *IterLong) Next() bool {

	if it.err != nil || it.finished {
		return false
	}

	var first bool
	if it.t == 0 {
		it.t = it.T0
		first = true
	}

	// read delta-of-delta
	dod, ok := it.dod()
	if !ok {
		return false
	}

	it.tDelta += uint32(dod)
	it.t = it.t + it.tDelta

	if first {
		// first point. read the float raw
		v, err := it.br.readBits(64)
		if err != nil {
			it.err = err
			return false
		}

		it.val = math.Float64frombits(v)
		return true
	}

	// read compressed value
	bit, err := it.br.readBit()
	if err != nil {
		it.err = err
		return false
	}

	if bit == zero {
		// it.val = it.val
	} else {
		bit, itErr := it.br.readBit()
		if itErr != nil {
			it.err = err
			return false
		}
		if bit == zero {
			// reuse leading/trailing zero bits
			// it.leading, it.trailing = it.leading, it.trailing
		} else {
			bits, err := it.br.readBits(5)
			if err != nil {
				it.err = err
				return false
			}
			it.leading = uint8(bits)

			bits, err = it.br.readBits(6)
			if err != nil {
				it.err = err
				return false
			}
			mbits := uint8(bits)
			// 0 significant bits here means we overflowed and we actually need 64; see comment in encoder
			if mbits == 0 {
				mbits = 64
			}
			it.trailing = 64 - it.leading - mbits
		}

		mbits := int(64 - it.leading - it.trailing)
		bits, err := it.br.readBits(mbits)
		if err != nil {
			it.err = err
			return false
		}
		vbits := math.Float64bits(it.val)
		vbits ^= (bits << it.trailing)
		it.val = math.Float64frombits(vbits)
	}

	return true
}

// Values at the current iterator position
func (it *IterLong) Values() (uint32, float64) {
	return it.t, it.val
}

// Err error at the current iterator position
func (it *IterLong) Err() error {
	return it.err
}

// MarshalBinary implements the encoding.BinaryMarshaler interface
func (s *SeriesLong) MarshalBinary() ([]byte, error) {
	buf := new(bytes.Buffer)
	em := &errMarshal{w: buf}
	em.write(s.T0)
	em.write(s.leading)
	em.write(s.T)
	em.write(s.tDelta)
	em.write(s.trailing)
	em.write(s.val)
	bStream, err := s.bw.MarshalBinary()
	if err != nil {
		return nil, err
	}
	em.write(bStream)
	if em.err != nil {
		return nil, em.err
	}
	return buf.Bytes(), nil
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface
func (s *SeriesLong) UnmarshalBinary(b []byte) error {
	buf := bytes.NewReader(b)
	em := &errMarshal{r: buf}
	em.read(&s.T0)
	em.read(&s.leading)
	em.read(&s.T)
	em.read(&s.tDelta)
	em.read(&s.trailing)
	em.read(&s.val)
	outBuf := make([]byte, buf.Len())
	em.read(outBuf)
	err := s.bw.UnmarshalBinary(outBuf)
	if err != nil {
		return err
	}
	if em.err != nil {
		return em.err
	}
	return nil
}
<|endoftext|>


<commit_before>package jcapi

import "fmt"

type JCSystem struct {
	Os                             string  `json:os`
	TemplateName                   string  `json:templateName`
	AllowSshRootLogin              bool    `json:allowSsgRootLogin`
	Id                             string  `json:id`
	LastContact                    string  `json:lastContact`
	RemoteIP                       string  `json:remoteIP`
	Active                         bool    `json:active`
	SshRootEnabled                 bool    `json:sshRootEnabled`
	AmazonInstanceID               string  `json:amazonInstanceID,omitempty`
	SshPassEnabled                 bool    `json:sshPassEnabled`
	Version                        string  `json:version`
	AgentVersion                   string  `json:agentVersion`
	AllowPublicKeyAuth             bool    `json:allowPublicKeyAuthentication`
	Organization                   string  `json:organization`
	Created                        string  `json:created`
	Arch                           string  `json:arch`
	SystemTimezone                 float64 `json:systemTimeZone`
	AllowSshPasswordAuthentication bool    `json:allowSshPasswordAuthentication`
	DisplayName                    string  `json:displayName`
	ModifySSHDConfig               bool    `json:modifySSHDConfig`
	AllowMultiFactorAuthentication bool    `json:allowMultiFactorAuthentication`
	Hostname                       string  `json:hostname`

	Patches               []string `json:patches`
	SshParamList          []string `json:sshParams`
	PatchAlarmList        []string `json:patchAlarms`
	NetworkInterfaceList  []string `json:networkInterfaces`
	ConnectionHistoryList []string `json:connectionHistory`

	Tags              []JCTag
	SshdParams        []JCSSHDParam
	NetworkInterfaces []JCNetworkInterface
}

type JCSSHDParam struct {
	Name  string `json:name`
	Value string `json:value`
}

type JCNetworkInterface struct {
	Name     string `json:name`
	Internal bool   `json:internal`
	Family   string `json:family`
	Address  string `json:address`
}

func SystemsToString(systems []JCSystem) string {
	returnVal := ""

	for _, system := range systems {
		returnVal += system.ToString()
	}

	return returnVal
}

func (jcsystem JCSystem) ToString() string {
	//
	// WARNING: Never output password via this method, it could be logged in clear text
	//
	//	returnVal := fmt.Sprintf("JCUSER: Id=[%s] - UserName=[%s] - FName/LName=[%s/%s] - Email=[%s] - sudo=[%t] - Uid=%s - Gid=%s - enableManagedUid=%t\n",
	//		jcuser.Id, jcuser.UserName, jcuser.FirstName, jcuser.LastName,
	//		jcuser.Email, jcuser.Sudo, jcuser.Uid, jcuser.Gid, jcuser.EnableManagedUid)
	//
	//	returnVal += fmt.Sprintf("JCUSER: ExternallyManaged=[%t] - ExternalDN=[%s] - ExternalSourceType=[%s]\n",
	//		jcuser.ExternallyManaged, jcuser.ExternalDN, jcuser.ExternalSourceType)
	//
	//	for _, tag := range jcuser.Tags {
	//		returnVal += fmt.Sprintf("\t%s\n", tag.ToString())
	//	}
	returnVal := "I'm a return val"
	return returnVal
}

func getJCSSHDParamFieldsFromInterface(fields map[string]interface{}, params *JCSSHDParam) {
	params.Name = fields["name"].(string)
	params.Value = fields["value"].(string)
}

func getJCSSHDParamFromArray(paramArray []interface{}) []JCSSHDParam {
	var returnVal []JCSSHDParam
	returnVal = make([]JCSSHDParam, len(paramArray))

	for idx, rec := range paramArray {
		getJCSSHDParamFieldsFromInterface(rec.(map[string]interface{}), &returnVal[idx])
	}

	return returnVal

}

func getJCNetworkInterfaceFieldsFromInterface(fields map[string]interface{}, nic *JCNetworkInterface) {
	nic.Address = fields["address"].(string)
	nic.Family = fields["family"].(string)
	nic.Internal = fields["internal"].(bool)
	nic.Name = fields["name"].(string)
}

func getJCNetworkInterfacesFromArray(nicArray []interface{}) []JCNetworkInterface {
	var returnVal []JCNetworkInterface
	returnVal = make([]JCNetworkInterface, len(nicArray))

	for idx, rec := range nicArray {
		getJCNetworkInterfaceFieldsFromInterface(rec.(map[string]interface{}), &returnVal[idx])
	}

	return returnVal
}

func getJCSystemFieldsFromInterface(fields map[string]interface{}, system *JCSystem) {
	system.Os = fields["os"].(string)
	system.TemplateName = fields["templateName"].(string)
	system.AllowSshRootLogin = fields["allowSshRootLogin"].(bool)
	system.Id = fields["id"].(string)
	system.LastContact = fields["lastContact"].(string)
	system.RemoteIP = fields["remoteIP"].(string)
	system.Active = fields["active"].(bool)
	system.SshRootEnabled = fields["sshRootEnabled"].(bool)
	system.SshPassEnabled = fields["sshPassEnabled"].(bool)
	system.Version = fields["version"].(string)
	system.AgentVersion = fields["agentVersion"].(string)
	system.AllowPublicKeyAuth = fields["allowPublicKeyAuthentication"].(bool)
	system.Organization = fields["organization"].(string)
	system.Created = fields["created"].(string)
	system.Arch = fields["arch"].(string)
	system.SystemTimezone = fields["systemTimezone"].(float64)
	system.AllowSshPasswordAuthentication = fields["allowSshPasswordAuthentication"].(bool)
	system.DisplayName = fields["displayName"].(string)
	system.ModifySSHDConfig = fields["modifySSHDConfig"].(bool)
	system.AllowMultiFactorAuthentication = fields["allowMultiFactorAuthentication"].(bool)
	system.Hostname = fields["hostname"].(string)

	system.SshdParams = getJCSSHDParamFromArray(fields["sshdParams"].([]interface{}))
	system.NetworkInterfaces = getJCNetworkInterfacesFromArray(fields["networkInterfaces"].([]interface{}))

	if _, exists := fields["amazonInstanceID"]; exists {
		system.AmazonInstanceID = fields["amazonInstanceID"].(string)
	}
}

func getJCSystemsFromInterface(systemInt interface{}) []JCSystem {

	var returnVal []JCSystem

	recMap := systemInt.(map[string]interface{})

	results := recMap["results"].([]interface{})

	returnVal = make([]JCSystem, len(results))

	for idx, result := range results {
		getJCSystemFieldsFromInterface(result.(map[string]interface{}), &returnVal[idx])
	}

	return returnVal
}

// Executes a search by hostname via the JumpCloud API
func (jc JCAPI) GetSystemByhostname(hostname string, withTags bool) ([]JCSystem, JCError) {
	var returnVal []JCSystem

	jcSystemRec, err := jc.Post("/search/systems", jc.hostnameFilter(hostname))
	if err != nil {
		return nil, fmt.Errorf("ERROR: Post to JumpCloud failed, err='%s'", err)
	}

	returnVal = getJCSystemsFromInterface(jcSystemRec)

	if withTags {
		tags, err := jc.GetAllTags()
		if err != nil {
			return nil, fmt.Errorf("ERROR: Could not get tags, err='%s'", err)
		}

		for idx, _ := range returnVal {
			returnVal[idx].AddJCTagsToSystem(tags)
		}
	}

	return returnVal, nil
}

func (jc JCAPI) GetSystemById(systemId string, withTags bool) (system JCSystem, err JCError) {
	url := fmt.Sprintf("/systems/%s", systemId)

	retVal, err := jc.Get(url)
	if err != nil {
		err = fmt.Errorf("ERROR: Could not get system by ID '%s', err='%s'", systemId, err)
	}

	if retVal != nil {
		getJCSystemFieldsFromInterface(retVal.(map[string]interface{}), &system)

		if withTags {
			// I should be able to use err below as the err return value, but there's
			// a compiler bug here in that it thinks a := of err is shadowed here,
			// even though tags should be the only variable declared with the :=
			tags, err2 := jc.GetAllTags()
			if err != nil {
				err = fmt.Errorf("ERROR: Could not get tags, err='%s'", err2)
				return
			}

			system.AddJCTagsToSystem(tags)
		}
	}

	return
}

//
//func (jc JCAPI) GetSystemUsers(withTags bool) (userList []JCUser, err JCError) {
//	var returnVal []JCUser
//
//	for skip := 0; skip == 0 || len(returnVal) == searchLimit; skip += searchSkipInterval {
//		url := fmt.Sprintf("/systemusers?sort=username&skip=%d&limit=%d", skip, searchLimit)
//
//		jcUserRec, err2 := jc.Get(url)
//		if err != nil {
//			return nil, fmt.Errorf("ERROR: Post to JumpCloud failed, err='%s'", err2)
//		}
//
//		// We really only care about the ID for the following call...
//		returnVal = getJCUsersFromInterface(jcUserRec)
//
//		for i, _ := range returnVal {
//			if returnVal[i].Id != "" {
//
//				//
//				// Get the rest of the user record, which includes details like
//				// the externalDN...
//				//
//				// We'll get all the tags one time later, so don't get the tags on this call...
//				//
//				// See above about the compiler error that requires me to use err2 instead of err below...
//				//
//				detailedUser, err2 := jc.GetSystemUserById(returnVal[i].Id, false)
//				if err != nil {
//					err = fmt.Errorf("ERROR: Could not get details for user ID '%s', err='%s'", returnVal[i].Id, err2)
//					return
//				}
//
//				if detailedUser.Id != "" {
//					userList = append(userList, detailedUser)
//				}
//			}
//		}
//	}
//
//	if withTags {
//		tags, err := jc.GetAllTags()
//		if err != nil {
//			return nil, fmt.Errorf("ERROR: Could not get tags, err='%s'", err)
//		}
//
//		for idx, _ := range userList {
//			userList[idx].AddJCTags(tags)
//		}
//	}
//
//	return
//}
//
////
//// Add or Update a new user to JumpCloud
////
//func (jc JCAPI) AddUpdateUser(op JCOp, user JCUser) (userId string, err JCError) {
//	if user.Password != "" {
//		user.PasswordDate = getTimeString()
//	}
//
//	data, err := json.Marshal(user)
//	if err != nil {
//		return "", fmt.Errorf("ERROR: Could not marshal JCUser object, err='%s'", err)
//	}
//
//	url := "/systemusers"
//	if op == Update {
//		url += "/" + user.Id
//	}
//
//	jcUserRec, err := jc.Do(MapJCOpToHTTP(op), url, data)
//	if err != nil {
//		return "", fmt.Errorf("ERROR: Could not post new JCUser object, err='%s'", err)
//	}
//
//	var returnUser JCUser
//	getJCUserFieldsFromInterface(jcUserRec.(map[string]interface{}), &returnUser)
//
//	if returnUser.Email != user.Email {
//		return "", fmt.Errorf("ERROR: JumpCloud did not return the same email - this should never happen!")
//	}
//
//	userId = returnUser.Id
//
//	return
//}

func (jc JCAPI) DeleteSystem(system JCSystem) JCError {
	_, err := jc.Delete(fmt.Sprintf("/system/%s", system.Id))
	if err != nil {
		return fmt.Errorf("ERROR: Could not delete system '%s': err='%s'", system.Hostname, err)
	}

	return nil
}
<commit_msg>get all systems<commit_after>package jcapi

import (
	"encoding/json"
	"fmt"
)

type JCSystem struct {
	Os                             string  `json:os`
	TemplateName                   string  `json:templateName`
	AllowSshRootLogin              bool    `json:allowSsgRootLogin`
	Id                             string  `json:id`
	LastContact                    string  `json:lastContact`
	RemoteIP                       string  `json:remoteIP`
	Active                         bool    `json:active`
	SshRootEnabled                 bool    `json:sshRootEnabled`
	AmazonInstanceID               string  `json:amazonInstanceID,omitempty`
	SshPassEnabled                 bool    `json:sshPassEnabled`
	Version                        string  `json:version`
	AgentVersion                   string  `json:agentVersion`
	AllowPublicKeyAuth             bool    `json:allowPublicKeyAuthentication`
	Organization                   string  `json:organization`
	Created                        string  `json:created`
	Arch                           string  `json:arch`
	SystemTimezone                 float64 `json:systemTimeZone`
	AllowSshPasswordAuthentication bool    `json:allowSshPasswordAuthentication`
	DisplayName                    string  `json:displayName`
	ModifySSHDConfig               bool    `json:modifySSHDConfig`
	AllowMultiFactorAuthentication bool    `json:allowMultiFactorAuthentication`
	Hostname                       string  `json:hostname`

	Patches               []string `json:patches`
	SshParamList          []string `json:sshParams`
	PatchAlarmList        []string `json:patchAlarms`
	NetworkInterfaceList  []string `json:networkInterfaces`
	ConnectionHistoryList []string `json:connectionHistory`

	Tags              []JCTag
	SshdParams        []JCSSHDParam
	NetworkInterfaces []JCNetworkInterface
}

type JCSSHDParam struct {
	Name  string `json:name`
	Value string `json:value`
}

type JCNetworkInterface struct {
	Name     string `json:name`
	Internal bool   `json:internal`
	Family   string `json:family`
	Address  string `json:address`
}

func SystemsToString(systems []JCSystem) string {
	returnVal := ""

	for _, system := range systems {
		returnVal += system.ToString()
	}

	return returnVal
}

func (jcsystem JCSystem) ToString() string {
	//
	// WARNING: Never output password via this method, it could be logged in clear text
	//
	//	returnVal := fmt.Sprintf("JCUSER: Id=[%s] - UserName=[%s] - FName/LName=[%s/%s] - Email=[%s] - sudo=[%t] - Uid=%s - Gid=%s - enableManagedUid=%t\n",
	//		jcuser.Id, jcuser.UserName, jcuser.FirstName, jcuser.LastName,
	//		jcuser.Email, jcuser.Sudo, jcuser.Uid, jcuser.Gid, jcuser.EnableManagedUid)
	//
	//	returnVal += fmt.Sprintf("JCUSER: ExternallyManaged=[%t] - ExternalDN=[%s] - ExternalSourceType=[%s]\n",
	//		jcuser.ExternallyManaged, jcuser.ExternalDN, jcuser.ExternalSourceType)
	//
	//	for _, tag := range jcuser.Tags {
	//		returnVal += fmt.Sprintf("\t%s\n", tag.ToString())
	//	}
	returnVal := "I'm a return val"
	return returnVal
}

func getJCSSHDParamFieldsFromInterface(fields map[string]interface{}, params *JCSSHDParam) {
	params.Name = fields["name"].(string)
	params.Value = fields["value"].(string)
}

func getJCSSHDParamFromArray(paramArray []interface{}) []JCSSHDParam {
	var returnVal []JCSSHDParam
	returnVal = make([]JCSSHDParam, len(paramArray))

	for idx, rec := range paramArray {
		getJCSSHDParamFieldsFromInterface(rec.(map[string]interface{}), &returnVal[idx])
	}

	return returnVal

}

func getJCNetworkInterfaceFieldsFromInterface(fields map[string]interface{}, nic *JCNetworkInterface) {
	nic.Address = fields["address"].(string)
	nic.Family = fields["family"].(string)
	nic.Internal = fields["internal"].(bool)
	nic.Name = fields["name"].(string)
}

func getJCNetworkInterfacesFromArray(nicArray []interface{}) []JCNetworkInterface {
	var returnVal []JCNetworkInterface
	returnVal = make([]JCNetworkInterface, len(nicArray))

	for idx, rec := range nicArray {
		getJCNetworkInterfaceFieldsFromInterface(rec.(map[string]interface{}), &returnVal[idx])
	}

	return returnVal
}

func getJCSystemFieldsFromInterface(fields map[string]interface{}, system *JCSystem) {
	system.Os = fields["os"].(string)
	system.TemplateName = fields["templateName"].(string)
	system.AllowSshRootLogin = fields["allowSshRootLogin"].(bool)
	system.Id = fields["id"].(string)
	system.LastContact = fields["lastContact"].(string)
	system.RemoteIP = fields["remoteIP"].(string)
	system.Active = fields["active"].(bool)
	system.SshRootEnabled = fields["sshRootEnabled"].(bool)
	system.SshPassEnabled = fields["sshPassEnabled"].(bool)
	system.Version = fields["version"].(string)
	system.AgentVersion = fields["agentVersion"].(string)
	system.AllowPublicKeyAuth = fields["allowPublicKeyAuthentication"].(bool)
	system.Organization = fields["organization"].(string)
	system.Created = fields["created"].(string)
	system.Arch = fields["arch"].(string)
	system.SystemTimezone = fields["systemTimezone"].(float64)
	system.AllowSshPasswordAuthentication = fields["allowSshPasswordAuthentication"].(bool)
	system.DisplayName = fields["displayName"].(string)
	system.ModifySSHDConfig = fields["modifySSHDConfig"].(bool)
	system.AllowMultiFactorAuthentication = fields["allowMultiFactorAuthentication"].(bool)
	system.Hostname = fields["hostname"].(string)

	system.SshdParams = getJCSSHDParamFromArray(fields["sshdParams"].([]interface{}))
	system.NetworkInterfaces = getJCNetworkInterfacesFromArray(fields["networkInterfaces"].([]interface{}))

	if _, exists := fields["amazonInstanceID"]; exists {
		system.AmazonInstanceID = fields["amazonInstanceID"].(string)
	}
}

func getJCSystemsFromInterface(systemInt interface{}) []JCSystem {

	var returnVal []JCSystem

	recMap := systemInt.(map[string]interface{})

	results := recMap["results"].([]interface{})

	returnVal = make([]JCSystem, len(results))

	for idx, result := range results {
		getJCSystemFieldsFromInterface(result.(map[string]interface{}), &returnVal[idx])
	}

	return returnVal
}

// Executes a search by hostname via the JumpCloud API
func (jc JCAPI) GetSystemByhostname(hostname string, withTags bool) ([]JCSystem, JCError) {
	var returnVal []JCSystem

	jcSystemRec, err := jc.Post("/search/systems", jc.hostnameFilter(hostname))
	if err != nil {
		return nil, fmt.Errorf("ERROR: Post to JumpCloud failed, err='%s'", err)
	}

	returnVal = getJCSystemsFromInterface(jcSystemRec)

	if withTags {
		tags, err := jc.GetAllTags()
		if err != nil {
			return nil, fmt.Errorf("ERROR: Could not get tags, err='%s'", err)
		}

		for idx, _ := range returnVal {
			returnVal[idx].AddJCTagsToSystem(tags)
		}
	}

	return returnVal, nil
}

func (jc JCAPI) GetSystemById(systemId string, withTags bool) (system JCSystem, err JCError) {
	url := fmt.Sprintf("/systems/%s", systemId)

	retVal, err := jc.Get(url)
	if err != nil {
		err = fmt.Errorf("ERROR: Could not get system by ID '%s', err='%s'", systemId, err)
	}

	if retVal != nil {
		getJCSystemFieldsFromInterface(retVal.(map[string]interface{}), &system)

		if withTags {
			// I should be able to use err below as the err return value, but there's
			// a compiler bug here in that it thinks a := of err is shadowed here,
			// even though tags should be the only variable declared with the :=
			tags, err2 := jc.GetAllTags()
			if err != nil {
				err = fmt.Errorf("ERROR: Could not get tags, err='%s'", err2)
				return
			}

			system.AddJCTagsToSystem(tags)
		}
	}

	return
}

func (jc JCAPI) GetSystems(withTags bool) ([]JCSystem, JCError) {
	var returnVal []JCSystem

	for skip := 0; skip == 0 || len(returnVal) == searchLimitl; skip += searchSkipInteval {
		url := fmt.Sprintf("/systems?sort=hostname&skip=%d&limit=%d", skip, searchLimit)

		jcSysRec, err2 := jc.Get(url)
		if err != nil {
			return nil, fmt.ErrorF("ERROR: Get to JumpCloud failed, err='%s'", err2)
		}

		returnVal = getJCSystemsFromInterface(jcSysRec)

		if withTags {
			tags, err := jc.GetAllTags()
			if err != nil {
				return nil, fmt.Errorf("ERROR: Could not get tags, err='%s'", err)
			}

			for idx, _ := range returnVal {
				returnVal[idx].AddJCTagsToSystem(tags)
			}
		}

		return returnVal, nil

	}
}

//
// Update a system
//
func (jc JCAPI) UpdateSystem(system JCSystem) (Id string, err JCError) {
	data, err := json.Marshal(system)
	if err != nil {
		return "", fmt.Errorf("ERROR: Could not marshal JCSystem object, err='%s'", err)
	}

	url := "/systems/" + system.Id

	jcSysRec, err := jc.Put(url, data)
	if err != nil {
		return "", fmt.Errorf("ERROR: Could not update JCSystem object, err='%s'", err)
	}

	var returnSystem JCSystem
	getJCSystemFieldsFromInterface(jcSysRec.(map[string]interface{}), &returnSystem)

	if returnUser.hostname != user.hostname {
		return "", fmt.Errorf("ERROR: JumpCloud did not return the same hostname - this should never happen!")
	}

	userId = returnUser.Id

	return
}

func (jc JCAPI) DeleteSystem(system JCSystem) JCError {
	_, err := jc.Delete(fmt.Sprintf("/system/%s", system.Id))
	if err != nil {
		return fmt.Errorf("ERROR: Could not delete system '%s': err='%s'", system.Hostname, err)
	}

	return nil
}
<|endoftext|>


<commit_before>package fetcher

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/Songmu/retry"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/model"
	"github.com/uber-go/zap"
	"golang.org/x/text/width"
	"gopkg.in/xmlpath.v2"
)

const (
	userAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html"
)

var redirectErrorFunc = func(req *http.Request, via []*http.Request) error {
	return http.ErrUseLastResponse
}

var (
	_                 = fmt.Print
	fetcherHTTPClient = &http.Client{
		Timeout:       5 * time.Second,
		CheckRedirect: redirectErrorFunc,
		Transport: &http.Transport{
			MaxIdleConns:        100,
			MaxIdleConnsPerHost: 10, // TODO: use `concurrency`
			Proxy:               http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).DialContext,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
		},
	}
	titleXPath      = xmlpath.MustCompile(`//title`)
	attributesXPath = xmlpath.MustCompile(`//div[@class='confirm low']/dl`)
	lessonXPath     = xmlpath.MustCompile(`//ul[@class='oneday']//li`)
	classAttrXPath  = xmlpath.MustCompile(`@class`)
)

type teacherLessons struct {
	teacher *model.Teacher
	lessons []*model.Lesson
}

type TeacherLessonFetcher struct {
	httpClient    *http.Client
	semaphore     chan struct{}
	cache         map[uint32]*teacherLessons
	cacheResponse bool
	logger        zap.Logger
	mCountries    *model.MCountries
}

func NewTeacherLessonFetcher(
	httpClient *http.Client, concurrency int, cacheResponse bool,
	mCountries *model.MCountries, log zap.Logger,
) *TeacherLessonFetcher {
	if httpClient == nil {
		httpClient = fetcherHTTPClient
	}
	if concurrency < 1 {
		concurrency = 1
	}
	if log == nil {
		log = zap.New(zap.NewJSONEncoder(zap.RFC3339Formatter("ts")))
	}
	semaphore := make(chan struct{}, concurrency)
	cache := make(map[uint32]*teacherLessons, 5000)
	return &TeacherLessonFetcher{
		httpClient:    httpClient,
		semaphore:     semaphore,
		cacheResponse: cacheResponse,
		cache:         cache,
		logger:        log,
		mCountries:    mCountries,
	}
}

func (fetcher *TeacherLessonFetcher) Fetch(teacherID uint32) (*model.Teacher, []*model.Lesson, error) {
	fetcher.semaphore <- struct{}{}
	defer func() {
		<-fetcher.semaphore
	}()

	teacher := model.NewTeacher(teacherID)
	var content io.ReadCloser
	err := retry.Retry(2, 300*time.Millisecond, func() error {
		var err error
		content, err = fetcher.fetchContent(teacher.URL())
		return err
	})
	defer content.Close()
	if err != nil {
		return nil, nil, err
	}
	return fetcher.parseHTML(teacher, content)
}

func (fetcher *TeacherLessonFetcher) fetchContent(url string) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return ioutil.NopCloser(strings.NewReader("")),
			errors.InternalWrapf(err, "Failed to create HTTP request: url=%v", url)
	}
	req.Header.Set("User-Agent", userAgent)
	resp, err := fetcher.httpClient.Do(req)
	if err != nil {
		return ioutil.NopCloser(strings.NewReader("")),
			errors.InternalWrapf(err, "Failed httpClient.Do(): url=%v", url)
	}

	switch resp.StatusCode {
	case http.StatusOK:
		return resp.Body, nil
	case http.StatusMovedPermanently, http.StatusFound:
		return resp.Body, errors.NotFoundf("Teacher not found: url=%v, status=%v", url, resp.StatusCode)
	default:
		return resp.Body, errors.Internalf(
			"Unknown error in fetchContent: url=%v, status=%v",
			url, resp.StatusCode,
		)
	}
}

func (fetcher *TeacherLessonFetcher) parseHTML(
	teacher *model.Teacher,
	html io.Reader,
) (*model.Teacher, []*model.Lesson, error) {
	// Check cache
	if c, ok := fetcher.cache[teacher.ID]; ok {
		return c.teacher, c.lessons, nil
	}

	root, err := xmlpath.ParseHTML(html)
	if err != nil {
		return nil, nil, err
	}

	// teacher name
	if title, ok := titleXPath.String(root); ok {
		teacher.Name = strings.Trim(strings.Split(title, "-")[0], " ")
	} else {
		return nil, nil, errors.Internalf("failed to fetch teacher's name: url=%v", teacher.URL)
	}

	// TODO: Wrap up as a method
	nameXPath := xmlpath.MustCompile(`./dt`)
	valueXPath := xmlpath.MustCompile(`./dd`)
	for iter := attributesXPath.Iter(root); iter.Next(); {
		node := iter.Node()
		name, ok := nameXPath.String(node)
		if !ok {
			fetcher.logger.Error(
				fmt.Sprintf("Failed to parse teacher attribute: name=%v", name),
				zap.Uint("teacherID", uint(teacher.ID)),
			)
			continue
		}
		value, ok := valueXPath.String(node)
		if !ok {
			fetcher.logger.Error(
				fmt.Sprintf("Failed to parse teacher attribute: name=%v, value=%v", name, value),
				zap.Uint("teacherID", uint(teacher.ID)),
			)
			continue
		}
		if err := fetcher.setTeacherAttribute(teacher, strings.TrimSpace(name), strings.TrimSpace(value)); err != nil {
			fetcher.logger.Error(
				fmt.Sprintf("Failed to setTeacherAttribute: name=%v, value=%v", name, value),
				zap.Uint("teacherID", uint(teacher.ID)),
			)
		}
		//fmt.Printf("name = %v, value = %v\n", strings.TrimSpace(name), strings.TrimSpace(value))
	}
	//fmt.Printf("teacher = %+v\n", teacher)

	dateRegexp := regexp.MustCompile(`([\d]+)月([\d]+)日(.+)`)
	lessons := make([]*model.Lesson, 0, 1000)
	now := time.Now().In(config.LocalTimezone())
	originalDate := time.Now().In(config.LocalTimezone()).Truncate(24 * time.Hour)
	date := originalDate
	// lessons
	for iter := lessonXPath.Iter(root); iter.Next(); {
		node := iter.Node()
		timeClass, ok := classAttrXPath.String(node)
		if !ok {
			continue
		}

		text := strings.Trim(node.String(), " ")

		//fmt.Printf("text = '%v', timeClass = '%v'\n", text, timeClass)
		fetcher.logger.Debug("Scraping as", zap.String("timeClass", timeClass), zap.String("text", text))

		// blank, available, reserved
		if timeClass == "date" {
			group := dateRegexp.FindStringSubmatch(text)
			if len(group) > 0 {
				month, day := MustInt(group[1]), MustInt(group[2])
				year := date.Year()
				if now.Month() == time.December && month == 1 {
					year = now.Year() + 1
				}
				originalDate = time.Date(
					year, time.Month(month), int(day),
					0, 0, 0, 0,
					config.LocalTimezone(),
				)
				date = originalDate
			}
		} else if strings.HasPrefix(timeClass, "t-") && text != "" {
			tmp := strings.Split(timeClass, "-")
			hour, minute := MustInt(tmp[1]), MustInt(tmp[2])
			if hour >= 24 {
				// Convert 24:30 -> 00:30
				hour -= 24
				if date.Unix() == originalDate.Unix() {
					// Set date to next day for 24:30
					date = date.Add(24 * time.Hour)
				}
			}
			dt := time.Date(
				date.Year(), date.Month(), date.Day(),
				hour, minute, 0, 0,
				config.LocalTimezone(),
			)
			status := model.LessonStatuses.MustValueForAlias(text)
			fetcher.logger.Debug(
				"lesson",
				zap.String("dt", dt.Format("2006-01-02 15:04")),
				zap.String("status", model.LessonStatuses.MustName(status)),
			)
			lessons = append(lessons, &model.Lesson{
				TeacherID: teacher.ID,
				Datetime:  dt,
				Status:    model.LessonStatuses.MustName(status),
			})
		} else {
			// nop
		}
	}

	// Set teacher lesson data to cache
	if fetcher.cache {
		fetcher.cache[teacher.ID] = &teacherLessons{teacher: teacher, lessons: lessons}
	}

	return teacher, lessons, nil
}

// TODO: Move to model
func (fetcher *TeacherLessonFetcher) setTeacherAttribute(teacher *model.Teacher, name string, value string) error {
	switch name {
	case "国籍":
		c, found := fetcher.mCountries.GetByNameJA(value)
		if !found {
			return errors.NotFoundf("No MCountries for %v", value)
		}
		teacher.CountryID = c.ID
	case "誕生日":
		value = width.Narrow.String(value)
		if strings.TrimSpace(value) == "" {
			teacher.Birthday = time.Time{}
		} else {
			t, err := time.Parse("2006-01-02", value)
			if err != nil {
				return err
			}
			teacher.Birthday = t
		}
	case "性別":
		switch value {
		case "男性":
			teacher.Gender = "male" // TODO: enum
		case "女性":
			teacher.Gender = "female"
		default:
			return errors.Internalf("Unknown gender for %v", value)
		}
	case "経歴":
		yoe := -1
		switch value {
		case "1年未満":
			yoe = 0
		case "3年以上":
			yoe = 4
		default:
			value = strings.Replace(value, "年", "", -1)
			if v, err := strconv.ParseInt(width.Narrow.String(value), 10, 32); err == nil {
				yoe = int(v)
			} else {
				return errors.InternalWrapf(err, "Failed to convert to number: %v", value)
			}
		}
		teacher.YearsOfExperience = uint8(yoe)
	}
	return nil
}

func (fetcher *TeacherLessonFetcher) Close() {
	close(fetcher.semaphore)
}

func MustInt(s string) int {
	i, err := strconv.ParseInt(s, 10, 32)
	if err != nil {
		panic(err)
	}
	return int(i)
}
<commit_msg>Rename cache field<commit_after>package fetcher

import (
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/Songmu/retry"
	"github.com/oinume/lekcije/server/config"
	"github.com/oinume/lekcije/server/errors"
	"github.com/oinume/lekcije/server/model"
	"github.com/uber-go/zap"
	"golang.org/x/text/width"
	"gopkg.in/xmlpath.v2"
)

const (
	userAgent = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html"
)

var redirectErrorFunc = func(req *http.Request, via []*http.Request) error {
	return http.ErrUseLastResponse
}

var (
	_                 = fmt.Print
	fetcherHTTPClient = &http.Client{
		Timeout:       5 * time.Second,
		CheckRedirect: redirectErrorFunc,
		Transport: &http.Transport{
			MaxIdleConns:        100,
			MaxIdleConnsPerHost: 10, // TODO: use `concurrency`
			Proxy:               http.ProxyFromEnvironment,
			DialContext: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).DialContext,
			IdleConnTimeout:       90 * time.Second,
			TLSHandshakeTimeout:   10 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
		},
	}
	titleXPath      = xmlpath.MustCompile(`//title`)
	attributesXPath = xmlpath.MustCompile(`//div[@class='confirm low']/dl`)
	lessonXPath     = xmlpath.MustCompile(`//ul[@class='oneday']//li`)
	classAttrXPath  = xmlpath.MustCompile(`@class`)
)

type teacherLessons struct {
	teacher *model.Teacher
	lessons []*model.Lesson
}

type TeacherLessonFetcher struct {
	httpClient *http.Client
	semaphore  chan struct{}
	cache      map[uint32]*teacherLessons
	caching    bool
	logger     zap.Logger
	mCountries *model.MCountries
}

func NewTeacherLessonFetcher(
	httpClient *http.Client, concurrency int, caching bool,
	mCountries *model.MCountries, log zap.Logger,
) *TeacherLessonFetcher {
	if httpClient == nil {
		httpClient = fetcherHTTPClient
	}
	if concurrency < 1 {
		concurrency = 1
	}
	if log == nil {
		log = zap.New(zap.NewJSONEncoder(zap.RFC3339Formatter("ts")))
	}
	semaphore := make(chan struct{}, concurrency)
	cache := make(map[uint32]*teacherLessons, 5000)
	return &TeacherLessonFetcher{
		httpClient: httpClient,
		semaphore:  semaphore,
		caching:    caching,
		cache:      cache,
		logger:     log,
		mCountries: mCountries,
	}
}

func (fetcher *TeacherLessonFetcher) Fetch(teacherID uint32) (*model.Teacher, []*model.Lesson, error) {
	fetcher.semaphore <- struct{}{}
	defer func() {
		<-fetcher.semaphore
	}()

	teacher := model.NewTeacher(teacherID)
	var content io.ReadCloser
	err := retry.Retry(2, 300*time.Millisecond, func() error {
		var err error
		content, err = fetcher.fetchContent(teacher.URL())
		return err
	})
	defer content.Close()
	if err != nil {
		return nil, nil, err
	}
	return fetcher.parseHTML(teacher, content)
}

func (fetcher *TeacherLessonFetcher) fetchContent(url string) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return ioutil.NopCloser(strings.NewReader("")),
			errors.InternalWrapf(err, "Failed to create HTTP request: url=%v", url)
	}
	req.Header.Set("User-Agent", userAgent)
	resp, err := fetcher.httpClient.Do(req)
	if err != nil {
		return ioutil.NopCloser(strings.NewReader("")),
			errors.InternalWrapf(err, "Failed httpClient.Do(): url=%v", url)
	}

	switch resp.StatusCode {
	case http.StatusOK:
		return resp.Body, nil
	case http.StatusMovedPermanently, http.StatusFound:
		return resp.Body, errors.NotFoundf("Teacher not found: url=%v, status=%v", url, resp.StatusCode)
	default:
		return resp.Body, errors.Internalf(
			"Unknown error in fetchContent: url=%v, status=%v",
			url, resp.StatusCode,
		)
	}
}

func (fetcher *TeacherLessonFetcher) parseHTML(
	teacher *model.Teacher,
	html io.Reader,
) (*model.Teacher, []*model.Lesson, error) {
	// Check cache
	if c, ok := fetcher.cache[teacher.ID]; ok {
		return c.teacher, c.lessons, nil
	}

	root, err := xmlpath.ParseHTML(html)
	if err != nil {
		return nil, nil, err
	}

	// teacher name
	if title, ok := titleXPath.String(root); ok {
		teacher.Name = strings.Trim(strings.Split(title, "-")[0], " ")
	} else {
		return nil, nil, errors.Internalf("failed to fetch teacher's name: url=%v", teacher.URL)
	}

	// TODO: Wrap up as a method
	nameXPath := xmlpath.MustCompile(`./dt`)
	valueXPath := xmlpath.MustCompile(`./dd`)
	for iter := attributesXPath.Iter(root); iter.Next(); {
		node := iter.Node()
		name, ok := nameXPath.String(node)
		if !ok {
			fetcher.logger.Error(
				fmt.Sprintf("Failed to parse teacher attribute: name=%v", name),
				zap.Uint("teacherID", uint(teacher.ID)),
			)
			continue
		}
		value, ok := valueXPath.String(node)
		if !ok {
			fetcher.logger.Error(
				fmt.Sprintf("Failed to parse teacher attribute: name=%v, value=%v", name, value),
				zap.Uint("teacherID", uint(teacher.ID)),
			)
			continue
		}
		if err := fetcher.setTeacherAttribute(teacher, strings.TrimSpace(name), strings.TrimSpace(value)); err != nil {
			fetcher.logger.Error(
				fmt.Sprintf("Failed to setTeacherAttribute: name=%v, value=%v", name, value),
				zap.Uint("teacherID", uint(teacher.ID)),
			)
		}
		//fmt.Printf("name = %v, value = %v\n", strings.TrimSpace(name), strings.TrimSpace(value))
	}
	//fmt.Printf("teacher = %+v\n", teacher)

	dateRegexp := regexp.MustCompile(`([\d]+)月([\d]+)日(.+)`)
	lessons := make([]*model.Lesson, 0, 1000)
	now := time.Now().In(config.LocalTimezone())
	originalDate := time.Now().In(config.LocalTimezone()).Truncate(24 * time.Hour)
	date := originalDate
	// lessons
	for iter := lessonXPath.Iter(root); iter.Next(); {
		node := iter.Node()
		timeClass, ok := classAttrXPath.String(node)
		if !ok {
			continue
		}

		text := strings.Trim(node.String(), " ")

		//fmt.Printf("text = '%v', timeClass = '%v'\n", text, timeClass)
		fetcher.logger.Debug("Scraping as", zap.String("timeClass", timeClass), zap.String("text", text))

		// blank, available, reserved
		if timeClass == "date" {
			group := dateRegexp.FindStringSubmatch(text)
			if len(group) > 0 {
				month, day := MustInt(group[1]), MustInt(group[2])
				year := date.Year()
				if now.Month() == time.December && month == 1 {
					year = now.Year() + 1
				}
				originalDate = time.Date(
					year, time.Month(month), int(day),
					0, 0, 0, 0,
					config.LocalTimezone(),
				)
				date = originalDate
			}
		} else if strings.HasPrefix(timeClass, "t-") && text != "" {
			tmp := strings.Split(timeClass, "-")
			hour, minute := MustInt(tmp[1]), MustInt(tmp[2])
			if hour >= 24 {
				// Convert 24:30 -> 00:30
				hour -= 24
				if date.Unix() == originalDate.Unix() {
					// Set date to next day for 24:30
					date = date.Add(24 * time.Hour)
				}
			}
			dt := time.Date(
				date.Year(), date.Month(), date.Day(),
				hour, minute, 0, 0,
				config.LocalTimezone(),
			)
			status := model.LessonStatuses.MustValueForAlias(text)
			fetcher.logger.Debug(
				"lesson",
				zap.String("dt", dt.Format("2006-01-02 15:04")),
				zap.String("status", model.LessonStatuses.MustName(status)),
			)
			lessons = append(lessons, &model.Lesson{
				TeacherID: teacher.ID,
				Datetime:  dt,
				Status:    model.LessonStatuses.MustName(status),
			})
		} else {
			// nop
		}
	}

	// Set teacher lesson data to cache
	if fetcher.caching {
		fetcher.cache[teacher.ID] = &teacherLessons{teacher: teacher, lessons: lessons}
	}

	return teacher, lessons, nil
}

// TODO: Move to model
func (fetcher *TeacherLessonFetcher) setTeacherAttribute(teacher *model.Teacher, name string, value string) error {
	switch name {
	case "国籍":
		c, found := fetcher.mCountries.GetByNameJA(value)
		if !found {
			return errors.NotFoundf("No MCountries for %v", value)
		}
		teacher.CountryID = c.ID
	case "誕生日":
		value = width.Narrow.String(value)
		if strings.TrimSpace(value) == "" {
			teacher.Birthday = time.Time{}
		} else {
			t, err := time.Parse("2006-01-02", value)
			if err != nil {
				return err
			}
			teacher.Birthday = t
		}
	case "性別":
		switch value {
		case "男性":
			teacher.Gender = "male" // TODO: enum
		case "女性":
			teacher.Gender = "female"
		default:
			return errors.Internalf("Unknown gender for %v", value)
		}
	case "経歴":
		yoe := -1
		switch value {
		case "1年未満":
			yoe = 0
		case "3年以上":
			yoe = 4
		default:
			value = strings.Replace(value, "年", "", -1)
			if v, err := strconv.ParseInt(width.Narrow.String(value), 10, 32); err == nil {
				yoe = int(v)
			} else {
				return errors.InternalWrapf(err, "Failed to convert to number: %v", value)
			}
		}
		teacher.YearsOfExperience = uint8(yoe)
	}
	return nil
}

func (fetcher *TeacherLessonFetcher) Close() {
	close(fetcher.semaphore)
}

func MustInt(s string) int {
	i, err := strconv.ParseInt(s, 10, 32)
	if err != nil {
		panic(err)
	}
	return int(i)
}
<|endoftext|>


<commit_before>package invokeAndQuery

import (
	"github.com/hyperledger/fabric/core/chaincode/shim"
	"build-chaincode/entities"
	"encoding/json"
	"errors"
	"build-chaincode/util"
	"crypto/sha256"
	"encoding/hex"
)

func CreateProject(stub shim.ChaincodeStubInterface, projectAsJson string) error {
	var project entities.Project
	err := json.Unmarshal([]byte(projectAsJson), &project)
	if err != nil {
		return errors.New("Error while unmarshalling trade, reason: " + err.Error())
	}

	// check if ProjectID exists, if not, the project cant be stored in the chaincode
	if project.ProjectID == "" {
		return errors.New("ProjectID is empty")
	}

	// TODO add validation for all the other fields

	// check if current user belongs to either the freelancer or the client
	userCompany, err := util.GetCompanyByCertificate(stub)
	if err != nil {
		return errors.New("Error while getting user company, reason: " + err.Error())
	}

	if userCompany.CompanyID != project.Freelancer && userCompany.CompanyID != project.Client {
		return errors.New("Current user doesn't belong to either the client or freelancer company")
	}

	user, err := util.GetCurrentBlockchainUser(stub)
	if err != nil {
		return errors.New("Error while getting user, reason" + err.Error())
	}

	project.CreatorID = user.UserID

	//marshall the trade so it can be stored in the chain.
	projectAsBytes, err := json.Marshal(project)
	if err != nil {
		return errors.New("Error in marshalling project" + err.Error())
	}

	//use the storeobject function to store the project
	util.StoreObjectInChain(stub, project.ProjectID, util.ProjecstIndexName, projectAsBytes)

	return nil
}

func GetProjects(stub shim.ChaincodeStubInterface) ([]entities.Project, error) {
	userCompany, err := util.GetCompanyByCertificate(stub)
	if err != nil {
		return []entities.Project{}, errors.New("Error while getting user company, reason: " + err.Error())
	}

	projectsIndex, err := util.GetIndex(stub, util.ProjecstIndexName)
	if err != nil {
		return []entities.Project{}, errors.New("Unable to retrieve projectsIndex, reason: " + err.Error())
	}

	projects := []entities.Project{}
	for _, projectID := range projectsIndex {
		projectAsBytes, err := stub.GetState(projectID)
		if err != nil {
			return []entities.Project{}, errors.New("Could not retrieve project for ID " + projectID + " reason: " + err.Error())
		}

		var project entities.Project
		err = json.Unmarshal(projectAsBytes, &project)
		if err != nil {
			return []entities.Project{}, errors.New("Error while unmarshalling projectAsBytes, reason: " + err.Error())
		}

		if project.Freelancer == userCompany.CompanyID || project.Client == userCompany.CompanyID {
			projects = append(projects, project)
		}
	}

	return projects, nil
}

func GetProjectByID(stub shim.ChaincodeStubInterface, projectID string) (entities.Project, error) {
	userCompany, err := util.GetCompanyByCertificate(stub)
	if err != nil {
		return entities.Project{}, errors.New("Error while getting user company, reason: " + err.Error())
	}

	projectAsBytes, err := stub.GetState(projectID)
	if err != nil {
		return entities.Project{}, errors.New("Could not retrieve project for ID " + projectID + " reason: " + err.Error())
	}

	var project entities.Project
	err = json.Unmarshal(projectAsBytes, &project)
	if err != nil {
		return entities.Project{}, errors.New("Error while unmarshalling projectAsBytes, reason: " + err.Error())
	}

	if project.Freelancer == userCompany.CompanyID || project.Client == userCompany.CompanyID {
		return project, nil
	}

	return entities.Project{}, nil
}

func SignAgreement(stub shim.ChaincodeStubInterface, projectID string, timestamp int64) error {
	project, err := GetProjectByID(stub, projectID)
	if err != nil {
		return errors.New("Error while getting project, reason: " + err.Error())
	}

	user, err := util.GetCurrentBlockchainUser(stub)
	if err != nil {
		return errors.New("Unable to retrieve user, reason: " + err.Error())
	}

	userCompany, err := util.GetCompanyByCertificate(stub)
	if err != nil {
		return errors.New("Error while getting user company, reason: " + err.Error())
	}

	if userCompany.CompanyID != project.Freelancer && userCompany.CompanyID != project.Client {
		return errors.New("Current user doesn't belong to either the client or freelancer company")
	}

	userRole, err := GetUserRole(userCompany.CompanyID, project)
	if err != nil {
		return err
	}

	projectHash := createProjectHash(project)

	signature := entities.Signature{Timestamp: timestamp, Hash: projectHash, UserID: user.UserID}

	if userRole == "freelancer" {
		project.Signatures.FreelancerSignature = signature;
	} else if userRole == "client" {
		project.Signatures.ClientSignature = signature;
	}

	if project.Signatures.FreelancerSignature.Hash == project.Signatures.ClientSignature.Hash {
		project.Signatures.SignedByBothParties = true;
		project.Status = "Signed"
	}

	return nil
}

func GetUserRole(companyID string, project entities.Project) (string, error) {
	if companyID == project.Freelancer {
		return "freelancer", nil
	} else if companyID == project.Client {
		return "client", nil
	} else {
		return "", errors.New("Can't find the role")
	}
}

type SubsetOfProject struct {
	ProjectName  string
	Freelancer   string
	Client       string
	StartDate    int64
	EndDate      int64
	HoursPerWeek int
}

func createProjectHash(project entities.Project) (string) {
	return calculate_hash([]string{project.ProjectName, project.Freelancer, project.Client, string(project.StartDate),
		string(project.EndDate), string(project.HoursPerWeek)})
}

func calculate_hash(args []string) string {
	var str = ""
	for _, v := range args {
		str += v
	}
	hasher := sha256.New()
	hasher.Write([]byte(str))
	return hex.EncodeToString(hasher.Sum(nil))
}<commit_msg>save signature<commit_after>package invokeAndQuery

import (
	"github.com/hyperledger/fabric/core/chaincode/shim"
	"build-chaincode/entities"
	"encoding/json"
	"errors"
	"build-chaincode/util"
	"crypto/sha256"
	"encoding/hex"
)

func CreateProject(stub shim.ChaincodeStubInterface, projectAsJson string) error {
	var project entities.Project
	err := json.Unmarshal([]byte(projectAsJson), &project)
	if err != nil {
		return errors.New("Error while unmarshalling trade, reason: " + err.Error())
	}

	// check if ProjectID exists, if not, the project cant be stored in the chaincode
	if project.ProjectID == "" {
		return errors.New("ProjectID is empty")
	}

	// TODO add validation for all the other fields

	// check if current user belongs to either the freelancer or the client
	userCompany, err := util.GetCompanyByCertificate(stub)
	if err != nil {
		return errors.New("Error while getting user company, reason: " + err.Error())
	}

	if userCompany.CompanyID != project.Freelancer && userCompany.CompanyID != project.Client {
		return errors.New("Current user doesn't belong to either the client or freelancer company")
	}

	user, err := util.GetCurrentBlockchainUser(stub)
	if err != nil {
		return errors.New("Error while getting user, reason" + err.Error())
	}

	project.CreatorID = user.UserID

	//marshall the trade so it can be stored in the chain.
	projectAsBytes, err := json.Marshal(project)
	if err != nil {
		return errors.New("Error in marshalling project" + err.Error())
	}

	//use the storeobject function to store the project
	util.StoreObjectInChain(stub, project.ProjectID, util.ProjecstIndexName, projectAsBytes)

	return nil
}

func GetProjects(stub shim.ChaincodeStubInterface) ([]entities.Project, error) {
	userCompany, err := util.GetCompanyByCertificate(stub)
	if err != nil {
		return []entities.Project{}, errors.New("Error while getting user company, reason: " + err.Error())
	}

	projectsIndex, err := util.GetIndex(stub, util.ProjecstIndexName)
	if err != nil {
		return []entities.Project{}, errors.New("Unable to retrieve projectsIndex, reason: " + err.Error())
	}

	projects := []entities.Project{}
	for _, projectID := range projectsIndex {
		projectAsBytes, err := stub.GetState(projectID)
		if err != nil {
			return []entities.Project{}, errors.New("Could not retrieve project for ID " + projectID + " reason: " + err.Error())
		}

		var project entities.Project
		err = json.Unmarshal(projectAsBytes, &project)
		if err != nil {
			return []entities.Project{}, errors.New("Error while unmarshalling projectAsBytes, reason: " + err.Error())
		}

		if project.Freelancer == userCompany.CompanyID || project.Client == userCompany.CompanyID {
			projects = append(projects, project)
		}
	}

	return projects, nil
}

func GetProjectByID(stub shim.ChaincodeStubInterface, projectID string) (entities.Project, error) {
	userCompany, err := util.GetCompanyByCertificate(stub)
	if err != nil {
		return entities.Project{}, errors.New("Error while getting user company, reason: " + err.Error())
	}

	projectAsBytes, err := stub.GetState(projectID)
	if err != nil {
		return entities.Project{}, errors.New("Could not retrieve project for ID " + projectID + " reason: " + err.Error())
	}

	var project entities.Project
	err = json.Unmarshal(projectAsBytes, &project)
	if err != nil {
		return entities.Project{}, errors.New("Error while unmarshalling projectAsBytes, reason: " + err.Error())
	}

	if project.Freelancer == userCompany.CompanyID || project.Client == userCompany.CompanyID {
		return project, nil
	}

	return entities.Project{}, nil
}

func SignAgreement(stub shim.ChaincodeStubInterface, projectID string, timestamp int64) error {
	project, err := GetProjectByID(stub, projectID)
	if err != nil {
		return errors.New("Error while getting project, reason: " + err.Error())
	}

	user, err := util.GetCurrentBlockchainUser(stub)
	if err != nil {
		return errors.New("Unable to retrieve user, reason: " + err.Error())
	}

	userCompany, err := util.GetCompanyByCertificate(stub)
	if err != nil {
		return errors.New("Error while getting user company, reason: " + err.Error())
	}

	if userCompany.CompanyID != project.Freelancer && userCompany.CompanyID != project.Client {
		return errors.New("Current user doesn't belong to either the client or freelancer company")
	}

	userRole, err := GetUserRole(userCompany.CompanyID, project)
	if err != nil {
		return err
	}

	projectHash := createProjectHash(project)

	signature := entities.Signature{Timestamp: timestamp, Hash: projectHash, UserID: user.UserID}

	if userRole == "freelancer" {
		project.Signatures.FreelancerSignature = signature;
	} else if userRole == "client" {
		project.Signatures.ClientSignature = signature;
	}

	if project.Signatures.FreelancerSignature.Hash == project.Signatures.ClientSignature.Hash {
		project.Signatures.SignedByBothParties = true;
		project.Status = "Signed"
	}

	project.LastUpdated = timestamp

	projectAsBytes, err := json.Marshal(project)
	if err != nil {
		return errors.New("Error in marshalling project" + err.Error())
	}

	err = stub.PutState(projectID, projectAsBytes)
	if err != nil {
		return errors.New("Could not store project" + err.Error())
	}

	return nil
}

func GetUserRole(companyID string, project entities.Project) (string, error) {
	if companyID == project.Freelancer {
		return "freelancer", nil
	} else if companyID == project.Client {
		return "client", nil
	} else {
		return "", errors.New("Can't find the role")
	}
}

type SubsetOfProject struct {
	ProjectName  string
	Freelancer   string
	Client       string
	StartDate    int64
	EndDate      int64
	HoursPerWeek int
}

func createProjectHash(project entities.Project) (string) {
	return calculate_hash([]string{project.ProjectName, project.Freelancer, project.Client, string(project.StartDate),
		string(project.EndDate), string(project.HoursPerWeek)})
}

func calculate_hash(args []string) string {
	var str = ""
	for _, v := range args {
		str += v
	}
	hasher := sha256.New()
	hasher.Write([]byte(str))
	return hex.EncodeToString(hasher.Sum(nil))
}<|endoftext|>