<commit_before>// Copyright (C) 2014 Constantin Schomburg <me@cschomburg.com>
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package xmpp

import (
	"fmt"
	"time"

	"github.com/agl/xmpp"

	"github.com/xconstruct/stark/core"
	"github.com/xconstruct/stark/proto"
	"github.com/xconstruct/stark/proto/natural"
)

var Module = core.Module{
	Name:        "xmpp",
	Version:     "1.0",
	NewInstance: NewInstance,
}

func init() {
	core.RegisterModule(Module)
}

type Config struct {
	Server   string
	User     string
	Domain   string
	Password string
}

type conversation struct {
	Remote      string
	Proto       *proto.Client
	LastMessage proto.Message
	Xmpp        *Client
}

type Client struct {
	cfg           Config
	ctx           *core.Context
	mux           *proto.Mux
	xmpp          *xmpp.Conn
	conversations map[string]*conversation
}

func New(ctx *core.Context) (*Client, error) {
	c := &Client{
		ctx:           ctx,
		conversations: make(map[string]*conversation, 0),
	}
	err := ctx.Config.Get("xmpp", &c.cfg)
	return c, err
}

func NewInstance(ctx *core.Context) (core.ModuleInstance, error) {
	return New(ctx)
}

func (c *Client) Enable() (err error) {
	c.mux = proto.NewMux()
	proto.Connect(c.ctx.Proto, c.mux)
	return c.connectXmpp()
}

func (c *Client) connectXmpp() (err error) {
	cfg := &xmpp.Config{}
	c.xmpp, err = xmpp.Dial(c.cfg.Server, c.cfg.User, c.cfg.Domain, c.cfg.Password, cfg)
	if err != nil {
		return err
	}
	go c.listen()
	return c.xmpp.SignalPresence("")
}

func (c *Client) reconnectLoop() {
	for {
		if err := c.connectXmpp(); err != nil {
			c.ctx.Log.Debugln("[xmpp] reconnect error:", err)
			time.Sleep(5 * time.Second)
		}
		c.ctx.Log.Infoln("[xmpp] reconnected")
		return
	}
}

func (c *Client) Disable() error {
	return nil
}

func (cv *conversation) handleProtoMessage(msg proto.Message) {
	cv.LastMessage = msg
	text := natural.FormatSimple(msg)
	cv.Xmpp.ctx.Log.Debugf("[xmpp] send '%s' to '%s'", text, cv.Remote)
	if err := cv.Xmpp.xmpp.Send(cv.Remote, text); err != nil {
		cv.Xmpp.ctx.Log.Errorln("[xmpp] send:", err)
	}
}

func (c *Client) listen() {
	for {
		stanza, err := c.xmpp.Next()
		if err != nil {
			c.reconnectLoop()
			return
		}

		switch v := stanza.Value.(type) {
		case *xmpp.ClientMessage:
			c.handleChatMessage(v)
		default:
			c.ctx.Log.Debugln("[xmpp] stanza", stanza.Name, v)
		}
	}
}

func (c *Client) newConversation(remote string) *conversation {
	ep := c.mux.NewEndpoint()
	client := proto.NewClient("xmpp-"+proto.GenerateId(), ep)
	cv := &conversation{
		Remote: remote,
		Proto:  client,
		Xmpp:   c,
	}
	if err := client.Subscribe("", "self", cv.handleProtoMessage); err != nil {
		c.ctx.Log.Errorln("[xmpp] new:", err)
	}
	c.conversations[cv.Remote] = cv
	return cv
}

func (c *Client) handleChatMessage(chat *xmpp.ClientMessage) {
	c.ctx.Log.Debugln("[xmpp] chat: ", chat)
	if chat.Body == "" {
		return
	}

	cv, ok := c.conversations[chat.From]
	if !ok {
		cv = c.newConversation(chat.From)
	}

	if chat.Body == "full" {
		text := fmt.Sprintf("%v", cv.LastMessage)
		if err := c.xmpp.Send(chat.From, text); err != nil {
			c.ctx.Log.Errorln("[xmpp] send:", err)
		}
		return
	}

	cv.Proto.Publish(proto.Message{
		Action: "natural/handle",
		Payload: map[string]interface{}{
			"text": chat.Body,
		},
	})
}
<commit_msg>XMPP: Pretty-format full JSON message.<commit_after>// Copyright (C) 2014 Constantin Schomburg <me@cschomburg.com>
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package xmpp

import (
	"encoding/json"
	"time"

	"github.com/agl/xmpp"

	"github.com/xconstruct/stark/core"
	"github.com/xconstruct/stark/proto"
	"github.com/xconstruct/stark/proto/natural"
)

var Module = core.Module{
	Name:        "xmpp",
	Version:     "1.0",
	NewInstance: NewInstance,
}

func init() {
	core.RegisterModule(Module)
}

type Config struct {
	Server   string
	User     string
	Domain   string
	Password string
}

type conversation struct {
	Remote      string
	Proto       *proto.Client
	LastMessage proto.Message
	Xmpp        *Client
}

type Client struct {
	cfg           Config
	ctx           *core.Context
	mux           *proto.Mux
	xmpp          *xmpp.Conn
	conversations map[string]*conversation
}

func New(ctx *core.Context) (*Client, error) {
	c := &Client{
		ctx:           ctx,
		conversations: make(map[string]*conversation, 0),
	}
	err := ctx.Config.Get("xmpp", &c.cfg)
	return c, err
}

func NewInstance(ctx *core.Context) (core.ModuleInstance, error) {
	return New(ctx)
}

func (c *Client) Enable() (err error) {
	c.mux = proto.NewMux()
	proto.Connect(c.ctx.Proto, c.mux)
	return c.connectXmpp()
}

func (c *Client) connectXmpp() (err error) {
	cfg := &xmpp.Config{}
	c.xmpp, err = xmpp.Dial(c.cfg.Server, c.cfg.User, c.cfg.Domain, c.cfg.Password, cfg)
	if err != nil {
		return err
	}
	go c.listen()
	return c.xmpp.SignalPresence("")
}

func (c *Client) reconnectLoop() {
	for {
		if err := c.connectXmpp(); err != nil {
			c.ctx.Log.Debugln("[xmpp] reconnect error:", err)
			time.Sleep(5 * time.Second)
		}
		c.ctx.Log.Infoln("[xmpp] reconnected")
		return
	}
}

func (c *Client) Disable() error {
	return nil
}

func (cv *conversation) handleProtoMessage(msg proto.Message) {
	cv.LastMessage = msg
	text := natural.FormatSimple(msg)
	cv.Xmpp.ctx.Log.Debugf("[xmpp] send '%s' to '%s'", text, cv.Remote)
	if err := cv.Xmpp.xmpp.Send(cv.Remote, text); err != nil {
		cv.Xmpp.ctx.Log.Errorln("[xmpp] send:", err)
	}
}

func (c *Client) listen() {
	for {
		stanza, err := c.xmpp.Next()
		if err != nil {
			c.reconnectLoop()
			return
		}

		switch v := stanza.Value.(type) {
		case *xmpp.ClientMessage:
			c.handleChatMessage(v)
		default:
			c.ctx.Log.Debugln("[xmpp] stanza", stanza.Name, v)
		}
	}
}

func (c *Client) newConversation(remote string) *conversation {
	ep := c.mux.NewEndpoint()
	client := proto.NewClient("xmpp-"+proto.GenerateId(), ep)
	cv := &conversation{
		Remote: remote,
		Proto:  client,
		Xmpp:   c,
	}
	if err := client.Subscribe("", "self", cv.handleProtoMessage); err != nil {
		c.ctx.Log.Errorln("[xmpp] new:", err)
	}
	c.conversations[cv.Remote] = cv
	return cv
}

func (c *Client) handleChatMessage(chat *xmpp.ClientMessage) {
	c.ctx.Log.Debugln("[xmpp] chat: ", chat)
	if chat.Body == "" {
		return
	}

	cv, ok := c.conversations[chat.From]
	if !ok {
		cv = c.newConversation(chat.From)
	}

	if chat.Body == "full" {
		text, err := json.MarshalIndent(cv.LastMessage, "", "    ")
		if err != nil {
			panic(err)
		}
		if err := c.xmpp.Send(chat.From, string(text)); err != nil {
			c.ctx.Log.Errorln("[xmpp] send:", err)
		}
		return
	}

	cv.Proto.Publish(proto.Message{
		Action: "natural/handle",
		Payload: map[string]interface{}{
			"text": chat.Body,
		},
	})
}
<|endoftext|>


<commit_before>// Package sfv provides a simple way of reading and verifying SFV (Simple File
// Verification) files.
package sfv

import (
	"bufio"
	"fmt"
	"hash/crc32"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
)

// Checksum represents a line in a SFV file, containing the filename, full path
// to the file and the CRC32 checksum
type Checksum struct {
	Filename string
	Path     string
	CRC32    uint32
}

// SFV contains all the checksums read from a SFV file.
type SFV struct {
	Checksums []Checksum
}

// Verify calculates the CRC32 of the associated file and returns true if the
// checksum is correct.
func (c *Checksum) Verify() (bool, error) {
	f, err := os.Open(c.Path)
	if err != nil {
		return false, err
	}
	defer f.Close()

	h := crc32.NewIEEE()
	reader := bufio.NewReader(f)
	buf := make([]byte, 4096)
	for {
		n, err := reader.Read(buf)
		if err != nil && err != io.EOF {
			return false, err
		}
		if n == 0 {
			break
		}
		h.Write(buf[:n])
	}
	success := h.Sum32() == c.CRC32
	return success, nil
}

// IsExist returns a boolean indicating if the file associated with the checksum
// exists
func (c *Checksum) IsExist() bool {
	_, err := os.Stat(c.Path)
	return err == nil
}

// Verify verifies all checksums contained in SFV and returns true if all
// checksums are correct.
func (s *SFV) Verify() (bool, error) {
	for _, c := range s.Checksums {
		ok, err := c.Verify()
		if err != nil {
			return false, err
		}
		if !ok {
			return false, nil
		}
	}
	return true, nil
}

// IsExist returns a boolean if all the files in SFV exists
func (s *SFV) IsExist() bool {
	for _, c := range s.Checksums {
		if !c.IsExist() {
			return false
		}
	}
	return true
}

func parseChecksum(dir string, s string) (*Checksum, error) {
	words := strings.SplitN(s, " ", 2)
	if len(words) != 2 {
		return nil, fmt.Errorf("expected 2 words, got %d", len(words))
	}

	filename := words[0]
	path := path.Join(dir, filename)
	crcString := strings.TrimLeft(words[1], " \t")
	crc32, err := strconv.ParseUint(crcString, 16, 32)
	if err != nil {
		return nil, err
	}
	// ParseUint will return error if number exceeds 32 bits
	return &Checksum{
		Path:     path,
		Filename: filename,
		CRC32:    uint32(crc32),
	}, nil
}

// Read reades a SFV file from filepath and creates a new SFV containing
// checksums parsed from the SFV file.
func Read(filepath string) (*SFV, error) {
	f, err := os.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	dir := path.Dir(filepath)
	checksums := []Checksum{}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.Trim(scanner.Text(), " \t")
		if len(line) == 0 || strings.HasPrefix(line, ";") {
			continue
		}
		checksum, err := parseChecksum(dir, line)
		if err != nil {
			return nil, err
		}
		checksums = append(checksums, *checksum)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return &SFV{Checksums: checksums}, nil
}
<commit_msg>Add path field<commit_after>// Package sfv provides a simple way of reading and verifying SFV (Simple File
// Verification) files.
package sfv

import (
	"bufio"
	"fmt"
	"hash/crc32"
	"io"
	"os"
	"path"
	"strconv"
	"strings"
)

// Checksum represents a line in a SFV file, containing the filename, full path
// to the file and the CRC32 checksum
type Checksum struct {
	Filename string
	Path     string
	CRC32    uint32
}

// SFV contains all the checksums read from a SFV file.
type SFV struct {
	Checksums []Checksum
	Path      string
}

// Verify calculates the CRC32 of the associated file and returns true if the
// checksum is correct.
func (c *Checksum) Verify() (bool, error) {
	f, err := os.Open(c.Path)
	if err != nil {
		return false, err
	}
	defer f.Close()

	h := crc32.NewIEEE()
	reader := bufio.NewReader(f)
	buf := make([]byte, 4096)
	for {
		n, err := reader.Read(buf)
		if err != nil && err != io.EOF {
			return false, err
		}
		if n == 0 {
			break
		}
		h.Write(buf[:n])
	}
	success := h.Sum32() == c.CRC32
	return success, nil
}

// IsExist returns a boolean indicating if the file associated with the checksum
// exists
func (c *Checksum) IsExist() bool {
	_, err := os.Stat(c.Path)
	return err == nil
}

// Verify verifies all checksums contained in SFV and returns true if all
// checksums are correct.
func (s *SFV) Verify() (bool, error) {
	for _, c := range s.Checksums {
		ok, err := c.Verify()
		if err != nil {
			return false, err
		}
		if !ok {
			return false, nil
		}
	}
	return true, nil
}

// IsExist returns a boolean if all the files in SFV exists
func (s *SFV) IsExist() bool {
	for _, c := range s.Checksums {
		if !c.IsExist() {
			return false
		}
	}
	return true
}

func parseChecksum(dir string, s string) (*Checksum, error) {
	words := strings.SplitN(s, " ", 2)
	if len(words) != 2 {
		return nil, fmt.Errorf("expected 2 words, got %d", len(words))
	}

	filename := words[0]
	path := path.Join(dir, filename)
	crcString := strings.TrimLeft(words[1], " \t")
	crc32, err := strconv.ParseUint(crcString, 16, 32)
	if err != nil {
		return nil, err
	}
	// ParseUint will return error if number exceeds 32 bits
	return &Checksum{
		Path:     path,
		Filename: filename,
		CRC32:    uint32(crc32),
	}, nil
}

// Read reades a SFV file from filepath and creates a new SFV containing
// checksums parsed from the SFV file.
func Read(filepath string) (*SFV, error) {
	f, err := os.Open(filepath)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	dir := path.Dir(filepath)
	checksums := []Checksum{}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.Trim(scanner.Text(), " \t")
		if len(line) == 0 || strings.HasPrefix(line, ";") {
			continue
		}
		checksum, err := parseChecksum(dir, line)
		if err != nil {
			return nil, err
		}
		checksums = append(checksums, *checksum)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return &SFV{
		Checksums: checksums,
		Path:      filepath,
	}, nil
}
<|endoftext|>


<commit_before>/* walter: a deployment pipeline template
 * Copyright (C) 2014 Recruit Technologies Co., Ltd. and contributors
 * (see CONTRIBUTORS.md)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package config

import (
	"container/list"
	"fmt"
	"reflect"
	"strings"

	"github.com/recruit-tech/walter/log"
	"github.com/recruit-tech/walter/messengers"
	"github.com/recruit-tech/walter/pipelines"
	"github.com/recruit-tech/walter/services"
	"github.com/recruit-tech/walter/stages"
)

func getStageTypeModuleName(stageType string) string {
	return strings.ToLower(stageType)
}

func Parse(configData *map[interface{}]interface{}) (*pipelines.Resources, error) {
	envs := NewEnvVariables()
	return ParseWithSpecifiedEnvs(configData, envs)
}

// TODO: make parser process a struct (for simplifying redundant functions and reducing the number of function parameters)
func ParseWithSpecifiedEnvs(configData *map[interface{}]interface{},
	envs *EnvVariables) (*pipelines.Resources, error) {
	// parse service block
	serviceOps, ok := (*configData)["service"].(map[interface{}]interface{})
	var repoService services.Service
	var err error
	if ok == true {
		log.Info("found \"service\" block")
		repoService, err = mapService(serviceOps, envs)
		if err != nil {
			return nil, err
		}
	} else {
		log.Info("not found \"service\" block")
		repoService, err = services.InitService("local")
		if err != nil {
			return nil, err
		}
	}

	// parse messenger block
	messengerOps, ok := (*configData)["messenger"].(map[interface{}]interface{})
	var messenger messengers.Messenger
	if ok == true {
		log.Info("found messenger block")
		messenger, err = mapMessenger(messengerOps, envs)
		if err != nil {
			return nil, err
		}
	} else {
		log.Info("not found messenger block")
		messenger, err = messengers.InitMessenger("fake")
		if err != nil {
			return nil, err
		}
	}

	// parse cleanup block
	var cleanup *pipelines.Pipeline = &pipelines.Pipeline{}
	cleanupData, ok := (*configData)["cleanup"].([]interface{})
	if ok == true {
		log.Info("found cleanup block")
		cleanupList, err := convertYamlMapToStages(cleanupData, envs)
		if err != nil {
			return nil, err
		}
		for stageItem := cleanupList.Front(); stageItem != nil; stageItem = stageItem.Next() {
			cleanup.AddStage(stageItem.Value.(stages.Stage))
		}
	} else {
		log.Info("not found cleanup block in the input file")
	}

	// parse pipeline block
	var pipeline *pipelines.Pipeline = &pipelines.Pipeline{}

	pipelineData, ok := (*configData)["pipeline"].([]interface{})
	if ok == false {
		return nil, fmt.Errorf("no pipeline block in the input file")
	}
	stageList, err := convertYamlMapToStages(pipelineData, envs)
	if err != nil {
		return nil, err
	}
	for stageItem := stageList.Front(); stageItem != nil; stageItem = stageItem.Next() {
		pipeline.AddStage(stageItem.Value.(stages.Stage))
	}
	var resources = &pipelines.Resources{Pipeline: pipeline, Cleanup: cleanup, Reporter: messenger, RepoService: repoService}

	return resources, nil
}

func mapMessenger(messengerMap map[interface{}]interface{}, envs *EnvVariables) (messengers.Messenger, error) {
	messengerType := messengerMap["type"].(string)
	log.Info("type of reporter is " + messengerType)
	messenger, err := messengers.InitMessenger(messengerType)
	if err != nil {
		return nil, err
	}
	newMessengerValue := reflect.ValueOf(messenger).Elem()
	newMessengerType := reflect.TypeOf(messenger).Elem()
	for i := 0; i < newMessengerType.NumField(); i++ {
		tagName := newMessengerType.Field(i).Tag.Get("config")
		for messengerOptKey, messengerOptVal := range messengerMap {
			if tagName == messengerOptKey {
				fieldVal := newMessengerValue.Field(i)
				if fieldVal.Type() == reflect.ValueOf("string").Type() {
					fieldVal.SetString(envs.Replace(messengerOptVal.(string)))
				}
			}
		}
	}

	return messenger, nil
}

func mapService(serviceMap map[interface{}]interface{}, envs *EnvVariables) (services.Service, error) {
	serviceType := serviceMap["type"].(string)
	log.Info("type of service is " + serviceType)
	service, err := services.InitService(serviceType)
	if err != nil {
		return nil, err
	}

	newServiceValue := reflect.ValueOf(service).Elem()
	newServiceType := reflect.TypeOf(service).Elem()
	for i := 0; i < newServiceType.NumField(); i++ {
		tagName := newServiceType.Field(i).Tag.Get("config")
		for serviceOptKey, serviceOptVal := range serviceMap {
			if tagName == serviceOptKey {
				fieldVal := newServiceValue.Field(i)
				if fieldVal.Type() == reflect.ValueOf("string").Type() {
					fieldVal.SetString(envs.Replace(serviceOptVal.(string)))
				}
			}
		}
	}
	return service, nil
}

func convertYamlMapToStages(yamlStageList []interface{}, envs *EnvVariables) (*list.List, error) {
	stages := list.New()
	for _, stageDetail := range yamlStageList {
		stage, err := mapStage(stageDetail.(map[interface{}]interface{}), envs)
		if err != nil {
			return nil, err
		}
		stages.PushBack(stage)
	}
	return stages, nil
}

func mapStage(stageMap map[interface{}]interface{}, envs *EnvVariables) (stages.Stage, error) {
	log.Debugf("%v", stageMap["run_after"])

	var stageType string = "command"
	if stageMap["type"] != nil {
		stageType = stageMap["type"].(string)
	} else if stageMap["stage_type"] != nil {
		log.Warn("found property \"stage_type\"")
		log.Warn("property \"stage_type\" is deprecated. please use \"type\" instead.")
		stageType = stageMap["stage_type"].(string)
	}
	stage, err := stages.InitStage(stageType)
	if err != nil {
		return nil, err
	}
	newStageValue := reflect.ValueOf(stage).Elem()
	newStageType := reflect.TypeOf(stage).Elem()

	if stageName := stageMap["name"]; stageName != nil {
		stage.SetStageName(stageMap["name"].(string))
	} else if stageName := stageMap["stage_name"]; stageName != nil {
		log.Warn("found property \"stage_name\"")
		log.Warn("property \"stage_name\" is deprecated. please use \"stage\" instead.")
		stage.SetStageName(stageMap["stage_name"].(string))
	}

	stageOpts := stages.NewStageOpts()

	if reportingFullOutput := stageMap["report_full_output"]; reportingFullOutput != nil {
		stageOpts.ReportingFullOutput = true
	}

	stage.SetStageOpts(*stageOpts)

	for i := 0; i < newStageType.NumField(); i++ {
		tagName := newStageType.Field(i).Tag.Get("config")
		is_replace := newStageType.Field(i).Tag.Get("is_replace")
		for stageOptKey, stageOptVal := range stageMap {
			if tagName == stageOptKey {
				if stageOptVal == nil {
					log.Warnf("stage option \"%s\" is not specified", stageOptKey)
				} else {
					setFieldVal(newStageValue.Field(i), stageOptVal, is_replace, envs)
				}
			}
		}
	}

	if runAfters := stageMap["run_after"]; runAfters != nil {
		for _, runAfter := range runAfters.([]interface{}) {
			childStage, err := mapStage(runAfter.(map[interface{}]interface{}), envs)
			if err != nil {
				return nil, err
			}
			stage.AddChildStage(childStage)
		}
	}
	return stage, nil
}

func setFieldVal(fieldVal reflect.Value, stageOptVal interface{}, is_replace string, envs *EnvVariables) {
	if fieldVal.Type() == reflect.ValueOf("string").Type() {
		if is_replace == "true" {
			fieldVal.SetString(envs.Replace(stageOptVal.(string)))
		} else {
			fieldVal.SetString(stageOptVal.(string))
		}
	}
}
<commit_msg>Implement `parallel' (just alias of run_after)<commit_after>/* walter: a deployment pipeline template
 * Copyright (C) 2014 Recruit Technologies Co., Ltd. and contributors
 * (see CONTRIBUTORS.md)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package config

import (
	"container/list"
	"fmt"
	"reflect"
	"strings"

	"github.com/recruit-tech/walter/log"
	"github.com/recruit-tech/walter/messengers"
	"github.com/recruit-tech/walter/pipelines"
	"github.com/recruit-tech/walter/services"
	"github.com/recruit-tech/walter/stages"
)

func getStageTypeModuleName(stageType string) string {
	return strings.ToLower(stageType)
}

func Parse(configData *map[interface{}]interface{}) (*pipelines.Resources, error) {
	envs := NewEnvVariables()
	return ParseWithSpecifiedEnvs(configData, envs)
}

// TODO: make parser process a struct (for simplifying redundant functions and reducing the number of function parameters)
func ParseWithSpecifiedEnvs(configData *map[interface{}]interface{},
	envs *EnvVariables) (*pipelines.Resources, error) {
	// parse service block
	serviceOps, ok := (*configData)["service"].(map[interface{}]interface{})
	var repoService services.Service
	var err error
	if ok == true {
		log.Info("found \"service\" block")
		repoService, err = mapService(serviceOps, envs)
		if err != nil {
			return nil, err
		}
	} else {
		log.Info("not found \"service\" block")
		repoService, err = services.InitService("local")
		if err != nil {
			return nil, err
		}
	}

	// parse messenger block
	messengerOps, ok := (*configData)["messenger"].(map[interface{}]interface{})
	var messenger messengers.Messenger
	if ok == true {
		log.Info("found messenger block")
		messenger, err = mapMessenger(messengerOps, envs)
		if err != nil {
			return nil, err
		}
	} else {
		log.Info("not found messenger block")
		messenger, err = messengers.InitMessenger("fake")
		if err != nil {
			return nil, err
		}
	}

	// parse cleanup block
	var cleanup *pipelines.Pipeline = &pipelines.Pipeline{}
	cleanupData, ok := (*configData)["cleanup"].([]interface{})
	if ok == true {
		log.Info("found cleanup block")
		cleanupList, err := convertYamlMapToStages(cleanupData, envs)
		if err != nil {
			return nil, err
		}
		for stageItem := cleanupList.Front(); stageItem != nil; stageItem = stageItem.Next() {
			cleanup.AddStage(stageItem.Value.(stages.Stage))
		}
	} else {
		log.Info("not found cleanup block in the input file")
	}

	// parse pipeline block
	var pipeline *pipelines.Pipeline = &pipelines.Pipeline{}

	pipelineData, ok := (*configData)["pipeline"].([]interface{})
	if ok == false {
		return nil, fmt.Errorf("no pipeline block in the input file")
	}
	stageList, err := convertYamlMapToStages(pipelineData, envs)
	if err != nil {
		return nil, err
	}
	for stageItem := stageList.Front(); stageItem != nil; stageItem = stageItem.Next() {
		pipeline.AddStage(stageItem.Value.(stages.Stage))
	}
	var resources = &pipelines.Resources{Pipeline: pipeline, Cleanup: cleanup, Reporter: messenger, RepoService: repoService}

	return resources, nil
}

func mapMessenger(messengerMap map[interface{}]interface{}, envs *EnvVariables) (messengers.Messenger, error) {
	messengerType := messengerMap["type"].(string)
	log.Info("type of reporter is " + messengerType)
	messenger, err := messengers.InitMessenger(messengerType)
	if err != nil {
		return nil, err
	}
	newMessengerValue := reflect.ValueOf(messenger).Elem()
	newMessengerType := reflect.TypeOf(messenger).Elem()
	for i := 0; i < newMessengerType.NumField(); i++ {
		tagName := newMessengerType.Field(i).Tag.Get("config")
		for messengerOptKey, messengerOptVal := range messengerMap {
			if tagName == messengerOptKey {
				fieldVal := newMessengerValue.Field(i)
				if fieldVal.Type() == reflect.ValueOf("string").Type() {
					fieldVal.SetString(envs.Replace(messengerOptVal.(string)))
				}
			}
		}
	}

	return messenger, nil
}

func mapService(serviceMap map[interface{}]interface{}, envs *EnvVariables) (services.Service, error) {
	serviceType := serviceMap["type"].(string)
	log.Info("type of service is " + serviceType)
	service, err := services.InitService(serviceType)
	if err != nil {
		return nil, err
	}

	newServiceValue := reflect.ValueOf(service).Elem()
	newServiceType := reflect.TypeOf(service).Elem()
	for i := 0; i < newServiceType.NumField(); i++ {
		tagName := newServiceType.Field(i).Tag.Get("config")
		for serviceOptKey, serviceOptVal := range serviceMap {
			if tagName == serviceOptKey {
				fieldVal := newServiceValue.Field(i)
				if fieldVal.Type() == reflect.ValueOf("string").Type() {
					fieldVal.SetString(envs.Replace(serviceOptVal.(string)))
				}
			}
		}
	}
	return service, nil
}

func convertYamlMapToStages(yamlStageList []interface{}, envs *EnvVariables) (*list.List, error) {
	stages := list.New()
	for _, stageDetail := range yamlStageList {
		stage, err := mapStage(stageDetail.(map[interface{}]interface{}), envs)
		if err != nil {
			return nil, err
		}
		stages.PushBack(stage)
	}
	return stages, nil
}

func mapStage(stageMap map[interface{}]interface{}, envs *EnvVariables) (stages.Stage, error) {
	log.Debugf("%v", stageMap["run_after"])

	var stageType string = "command"
	if stageMap["type"] != nil {
		stageType = stageMap["type"].(string)
	} else if stageMap["stage_type"] != nil {
		log.Warn("found property \"stage_type\"")
		log.Warn("property \"stage_type\" is deprecated. please use \"type\" instead.")
		stageType = stageMap["stage_type"].(string)
	}
	stage, err := stages.InitStage(stageType)
	if err != nil {
		return nil, err
	}
	newStageValue := reflect.ValueOf(stage).Elem()
	newStageType := reflect.TypeOf(stage).Elem()

	if stageName := stageMap["name"]; stageName != nil {
		stage.SetStageName(stageMap["name"].(string))
	} else if stageName := stageMap["stage_name"]; stageName != nil {
		log.Warn("found property \"stage_name\"")
		log.Warn("property \"stage_name\" is deprecated. please use \"stage\" instead.")
		stage.SetStageName(stageMap["stage_name"].(string))
	}

	stageOpts := stages.NewStageOpts()

	if reportingFullOutput := stageMap["report_full_output"]; reportingFullOutput != nil {
		stageOpts.ReportingFullOutput = true
	}

	stage.SetStageOpts(*stageOpts)

	for i := 0; i < newStageType.NumField(); i++ {
		tagName := newStageType.Field(i).Tag.Get("config")
		is_replace := newStageType.Field(i).Tag.Get("is_replace")
		for stageOptKey, stageOptVal := range stageMap {
			if tagName == stageOptKey {
				if stageOptVal == nil {
					log.Warnf("stage option \"%s\" is not specified", stageOptKey)
				} else {
					setFieldVal(newStageValue.Field(i), stageOptVal, is_replace, envs)
				}
			}
		}
	}

	runAfters := stageMap["run_after"]
	if runAfters == nil {
		runAfters = stageMap["parallel"]
	}

	if runAfters != nil {
		for _, runAfter := range runAfters.([]interface{}) {
			childStage, err := mapStage(runAfter.(map[interface{}]interface{}), envs)
			if err != nil {
				return nil, err
			}
			stage.AddChildStage(childStage)
		}
	}
	return stage, nil
}

func setFieldVal(fieldVal reflect.Value, stageOptVal interface{}, is_replace string, envs *EnvVariables) {
	if fieldVal.Type() == reflect.ValueOf("string").Type() {
		if is_replace == "true" {
			fieldVal.SetString(envs.Replace(stageOptVal.(string)))
		} else {
			fieldVal.SetString(stageOptVal.(string))
		}
	}
}
<|endoftext|>


<commit_before>package zlib

import (
	"errors"
	"fmt"
	"net"
	"regexp"
	"strconv"
	"time"

	"github.com/zmap/ztools/x509"
	"github.com/zmap/ztools/ztls"
)

var smtpEndRegex = regexp.MustCompile(`(?:\r\n)|^[0-9]{3} .+\r\n$`)
var pop3EndRegex = regexp.MustCompile(`(?:\r\n\.\r\n$)|(?:\r\n$)`)
var imapStatusEndRegex = regexp.MustCompile(`\r\n$`)

const (
	SMTP_COMMAND = "STARTTLS\r\n"
	POP3_COMMAND = "STLS\r\n"
	IMAP_COMMAND = "a001 STARTTLS\r\n"
)

// Implements the net.Conn interface
type Conn struct {
	// Underlying network connection
	conn    net.Conn
	tlsConn *ztls.Conn
	isTls   bool

	// Max TLS version
	maxTlsVersion uint16

	// Keep track of state / network operations
	operations []ConnectionEvent

	// Cache the deadlines so we can reapply after TLS handshake
	readDeadline  time.Time
	writeDeadline time.Time

	caPool *x509.CertPool

	onlyCBC      bool
	onlySchannel bool

	domain string

	// Encoding type
	ReadEncoding string
}

func (c *Conn) getUnderlyingConn() net.Conn {
	if c.isTls {
		return c.tlsConn
	}
	return c.conn
}

func (c *Conn) SetCBCOnly() {
	c.onlyCBC = true
}

func (c *Conn) SetSChannelOnly() {
	c.onlySchannel = true
}

func (c *Conn) SetCAPool(pool *x509.CertPool) {
	c.caPool = pool
}

func (c *Conn) SetDomain(domain string) {
	c.domain = domain
}

// Layer in the regular conn methods
func (c *Conn) LocalAddr() net.Addr {
	return c.getUnderlyingConn().LocalAddr()
}

func (c *Conn) RemoteAddr() net.Addr {
	return c.getUnderlyingConn().RemoteAddr()
}

func (c *Conn) SetDeadline(t time.Time) error {
	c.readDeadline = t
	c.writeDeadline = t
	return c.getUnderlyingConn().SetDeadline(t)
}

func (c *Conn) SetReadDeadline(t time.Time) error {
	c.readDeadline = t
	return c.getUnderlyingConn().SetReadDeadline(t)
}

func (c *Conn) SetWriteDeadline(t time.Time) error {
	c.writeDeadline = t
	return c.getUnderlyingConn().SetWriteDeadline(t)
}

// Delegate here, but record all the things
func (c *Conn) Write(b []byte) (int, error) {
	n, err := c.getUnderlyingConn().Write(b)
	w := WriteEvent{Sent: b}
	event := ConnectionEvent{
		Data:  &w,
		Error: err,
	}
	c.operations = append(c.operations, event)
	return n, err
}

func (c *Conn) Read(b []byte) (int, error) {
	n, err := c.getUnderlyingConn().Read(b)
	r := ReadEvent{
		Response: b[0:n],
		encoding: c.ReadEncoding,
	}
	c.appendEvent(&r, err)
	return n, err
}

func (c *Conn) Close() error {
	return c.getUnderlyingConn().Close()
}

// Extra method - Do a TLS Handshake and record progress
func (c *Conn) TLSHandshake() error {
	if c.isTls {
		return fmt.Errorf(
			"Attempted repeat handshake with remote host %s",
			c.RemoteAddr().String())
	}
	tlsConfig := new(ztls.Config)
	tlsConfig.InsecureSkipVerify = true
	tlsConfig.MinVersion = ztls.VersionSSL30
	tlsConfig.MaxVersion = c.maxTlsVersion
	tlsConfig.RootCAs = c.caPool
	if c.domain != "" {
		tlsConfig.ServerName = c.domain
	}
	if c.onlyCBC {
		tlsConfig.CipherSuites = ztls.CBCSuiteIDList
	}
	if c.onlySchannel {
		tlsConfig.CipherSuites = ztls.SChannelSuites
	}
	c.tlsConn = ztls.Client(c.conn, tlsConfig)
	c.tlsConn.SetReadDeadline(c.readDeadline)
	c.tlsConn.SetWriteDeadline(c.writeDeadline)
	c.isTls = true
	err := c.tlsConn.Handshake()
	hl := c.tlsConn.GetHandshakeLog()
	ts := TLSHandshakeEvent{handshakeLog: hl}
	event := ConnectionEvent{
		Data:  &ts,
		Error: err,
	}
	c.operations = append(c.operations, event)
	return err
}

func (c *Conn) sendStartTLSCommand(command string) error {
	// Don't doublehandshake
	if c.isTls {
		return fmt.Errorf(
			"Attempt STARTTLS after TLS handshake with remote host %s",
			c.RemoteAddr().String())
	}
	// Send the STARTTLS message
	starttls := []byte(command)
	_, err := c.conn.Write(starttls)
	return err
}

// Do a STARTTLS handshake
func (c *Conn) SMTPStartTLSHandshake() error {
	// Make the state
	ss := StartTLSEvent{Command: SMTP_COMMAND}

	// Send the command
	if err := c.sendStartTLSCommand(SMTP_COMMAND); err != nil {
		c.appendEvent(&ss, err)
		return err
	}
	// Read the response on a successful send
	buf := make([]byte, 256)
	n, err := c.readSmtpResponse(buf)
	ss.Response = string(buf[0:n])

	// Actually check return code
	if n < 5 {
		err = errors.New("Server did not indicate support for STARTTLS")
	}
	if err == nil {
		var ret int
		ret, err = strconv.Atoi(ss.Response[0:3])
		if err != nil || ret < 200 || ret >= 300 {
			err = errors.New("Bad return code for STARTTLS")
		}
	}

	// Record everything no matter the result
	c.appendEvent(&ss, err)

	// Stop if we failed already
	if err != nil {
		return err
	}

	// Successful so far, attempt to do the actual handshake
	return c.TLSHandshake()
}

func (c *Conn) POP3StartTLSHandshake() error {
	ss := StartTLSEvent{Command: POP3_COMMAND}
	if err := c.sendStartTLSCommand(POP3_COMMAND); err != nil {
		c.appendEvent(&ss, err)
		return err
	}

	buf := make([]byte, 512)
	n, err := c.readPop3Response(buf)
	ss.Response = string(buf[0:n])
	c.appendEvent(&ss, err)

	if err != nil {
		return err
	}
	return c.TLSHandshake()
}

func (c *Conn) IMAPStartTLSHandshake() error {
	ss := StartTLSEvent{Command: IMAP_COMMAND}
	if err := c.sendStartTLSCommand(IMAP_COMMAND); err != nil {
		c.appendEvent(&ss, err)
		return err
	}

	buf := make([]byte, 512)
	n, err := c.readImapStatusResponse(buf)
	ss.Response = string(buf[0:n])
	c.appendEvent(&ss, err)

	if err != nil {
		return err
	}
	return c.TLSHandshake()
}

func (c *Conn) readUntilRegex(res []byte, expr *regexp.Regexp) (int, error) {
	buf := res[0:]
	length := 0
	for finished := false; !finished; {
		n, err := c.getUnderlyingConn().Read(buf)
		length += n
		if err != nil {
			return length, err
		}
		if expr.Match(res[0:length]) {
			finished = true
		}
		if length == len(res) {
			return length, errors.New("Not enough buffer space")
		}
		buf = res[length:]
	}
	return length, nil
}

func (c *Conn) readSmtpResponse(res []byte) (int, error) {
	return c.readUntilRegex(res, smtpEndRegex)
}

func (c *Conn) SMTPBanner(b []byte) (int, error) {
	n, err := c.readSmtpResponse(b)
	mb := MailBannerEvent{}
	mb.Banner = string(b[0:n])
	c.appendEvent(&mb, err)
	return n, err
}

func (c *Conn) EHLO(domain string) error {
	cmd := []byte("EHLO " + domain + "\r\n")
	ee := EHLOEvent{}
	if _, err := c.getUnderlyingConn().Write(cmd); err != nil {
		c.appendEvent(&ee, err)
		return err
	}

	buf := make([]byte, 512)
	n, err := c.readSmtpResponse(buf)
	ee.Response = string(buf[0:n])
	c.appendEvent(&ee, err)
	return err
}

func (c *Conn) SMTPHelp() error {
	cmd := []byte("HELP\r\n")
	h := new(SMTPHelpEvent)
	if _, err := c.getUnderlyingConn().Write(cmd); err != nil {
		c.appendEvent(h, err)
		return err
	}
	buf := make([]byte, 512)
	n, err := c.readSmtpResponse(buf)
	h.Response = string(buf[0:n])
	c.appendEvent(h, err)
	return err
}

func (c *Conn) readPop3Response(res []byte) (int, error) {
	return c.readUntilRegex(res, pop3EndRegex)
}

func (c *Conn) POP3Banner(b []byte) (int, error) {
	n, err := c.readPop3Response(b)
	mb := MailBannerEvent{
		Banner: string(b[0:n]),
	}
	c.appendEvent(&mb, err)
	return n, err
}

func (c *Conn) readImapStatusResponse(res []byte) (int, error) {
	return c.readUntilRegex(res, imapStatusEndRegex)
}

func (c *Conn) IMAPBanner(b []byte) (int, error) {
	n, err := c.readImapStatusResponse(b)
	mb := MailBannerEvent{
		Banner: string(b[0:n]),
	}
	c.appendEvent(&mb, err)
	return n, err
}

func (c *Conn) CheckHeartbleed(b []byte) (int, error) {
	if !c.isTls {
		return 0, fmt.Errorf(
			"Must perform TLS handshake before sending Heartbleed probe to %s",
			c.RemoteAddr().String())
	}
	n, err := c.tlsConn.CheckHeartbleed(b)
	hb := c.tlsConn.GetHeartbleedLog()
	if err == ztls.HeartbleedError {
		err = nil
	}
	event := ConnectionEvent{
		Data:  &HeartbleedEvent{heartbleedLog: hb},
		Error: err,
	}
	c.operations = append(c.operations, event)
	return n, err
}

func (c *Conn) SendModbusEcho() (int, error) {
	req := ModbusRequest{
		Function: ModbusFunctionEncapsulatedInterface,
		Data: []byte{
			0x0E, // read device info
			0x01, // product code
			0x00, // object id, should always be 0 in initial request
		},
	}

	event := new(ModbusEvent)
	data, err := req.MarshalBinary()
	w := 0
	for w < len(data) {
		written, err := c.getUnderlyingConn().Write(data[w:]) // TODO verify write
		w += written
		if err != nil {
			c.appendEvent(event, err)
			return w, errors.New("Could not write modbus request")
		}
	}

	res, err := c.GetModbusResponse()
	event.Length = res.Length
	event.UnitID = res.UnitID
	event.Function = res.Function
	event.Response = res.Data
	event.ParseSelf()
	// make sure the whole thing gets appended to the operation log
	c.appendEvent(event, err)
	return w, err
}

func (c *Conn) States() []ConnectionEvent {
	return c.operations
}

func (c *Conn) appendEvent(data EventData, err error) {
	event := ConnectionEvent{
		Data:  data,
		Error: err,
	}
	c.operations = append(c.operations, event)
}
<commit_msg>POP3 - Only use STARTTLS when server supports it<commit_after>package zlib

import (
	"errors"
	"fmt"
	"net"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/zmap/ztools/x509"
	"github.com/zmap/ztools/ztls"
)

var smtpEndRegex = regexp.MustCompile(`(?:\r\n)|^[0-9]{3} .+\r\n$`)
var pop3EndRegex = regexp.MustCompile(`(?:\r\n\.\r\n$)|(?:\r\n$)`)
var imapStatusEndRegex = regexp.MustCompile(`\r\n$`)

const (
	SMTP_COMMAND = "STARTTLS\r\n"
	POP3_COMMAND = "STLS\r\n"
	IMAP_COMMAND = "a001 STARTTLS\r\n"
)

// Implements the net.Conn interface
type Conn struct {
	// Underlying network connection
	conn    net.Conn
	tlsConn *ztls.Conn
	isTls   bool

	// Max TLS version
	maxTlsVersion uint16

	// Keep track of state / network operations
	operations []ConnectionEvent

	// Cache the deadlines so we can reapply after TLS handshake
	readDeadline  time.Time
	writeDeadline time.Time

	caPool *x509.CertPool

	onlyCBC      bool
	onlySchannel bool

	domain string

	// Encoding type
	ReadEncoding string
}

func (c *Conn) getUnderlyingConn() net.Conn {
	if c.isTls {
		return c.tlsConn
	}
	return c.conn
}

func (c *Conn) SetCBCOnly() {
	c.onlyCBC = true
}

func (c *Conn) SetSChannelOnly() {
	c.onlySchannel = true
}

func (c *Conn) SetCAPool(pool *x509.CertPool) {
	c.caPool = pool
}

func (c *Conn) SetDomain(domain string) {
	c.domain = domain
}

// Layer in the regular conn methods
func (c *Conn) LocalAddr() net.Addr {
	return c.getUnderlyingConn().LocalAddr()
}

func (c *Conn) RemoteAddr() net.Addr {
	return c.getUnderlyingConn().RemoteAddr()
}

func (c *Conn) SetDeadline(t time.Time) error {
	c.readDeadline = t
	c.writeDeadline = t
	return c.getUnderlyingConn().SetDeadline(t)
}

func (c *Conn) SetReadDeadline(t time.Time) error {
	c.readDeadline = t
	return c.getUnderlyingConn().SetReadDeadline(t)
}

func (c *Conn) SetWriteDeadline(t time.Time) error {
	c.writeDeadline = t
	return c.getUnderlyingConn().SetWriteDeadline(t)
}

// Delegate here, but record all the things
func (c *Conn) Write(b []byte) (int, error) {
	n, err := c.getUnderlyingConn().Write(b)
	w := WriteEvent{Sent: b}
	event := ConnectionEvent{
		Data:  &w,
		Error: err,
	}
	c.operations = append(c.operations, event)
	return n, err
}

func (c *Conn) Read(b []byte) (int, error) {
	n, err := c.getUnderlyingConn().Read(b)
	r := ReadEvent{
		Response: b[0:n],
		encoding: c.ReadEncoding,
	}
	c.appendEvent(&r, err)
	return n, err
}

func (c *Conn) Close() error {
	return c.getUnderlyingConn().Close()
}

// Extra method - Do a TLS Handshake and record progress
func (c *Conn) TLSHandshake() error {
	if c.isTls {
		return fmt.Errorf(
			"Attempted repeat handshake with remote host %s",
			c.RemoteAddr().String())
	}
	tlsConfig := new(ztls.Config)
	tlsConfig.InsecureSkipVerify = true
	tlsConfig.MinVersion = ztls.VersionSSL30
	tlsConfig.MaxVersion = c.maxTlsVersion
	tlsConfig.RootCAs = c.caPool
	if c.domain != "" {
		tlsConfig.ServerName = c.domain
	}
	if c.onlyCBC {
		tlsConfig.CipherSuites = ztls.CBCSuiteIDList
	}
	if c.onlySchannel {
		tlsConfig.CipherSuites = ztls.SChannelSuites
	}
	c.tlsConn = ztls.Client(c.conn, tlsConfig)
	c.tlsConn.SetReadDeadline(c.readDeadline)
	c.tlsConn.SetWriteDeadline(c.writeDeadline)
	c.isTls = true
	err := c.tlsConn.Handshake()
	hl := c.tlsConn.GetHandshakeLog()
	ts := TLSHandshakeEvent{handshakeLog: hl}
	event := ConnectionEvent{
		Data:  &ts,
		Error: err,
	}
	c.operations = append(c.operations, event)
	return err
}

func (c *Conn) sendStartTLSCommand(command string) error {
	// Don't doublehandshake
	if c.isTls {
		return fmt.Errorf(
			"Attempt STARTTLS after TLS handshake with remote host %s",
			c.RemoteAddr().String())
	}
	// Send the STARTTLS message
	starttls := []byte(command)
	_, err := c.conn.Write(starttls)
	return err
}

// Do a STARTTLS handshake
func (c *Conn) SMTPStartTLSHandshake() error {
	// Make the state
	ss := StartTLSEvent{Command: SMTP_COMMAND}

	// Send the command
	if err := c.sendStartTLSCommand(SMTP_COMMAND); err != nil {
		c.appendEvent(&ss, err)
		return err
	}
	// Read the response on a successful send
	buf := make([]byte, 256)
	n, err := c.readSmtpResponse(buf)
	ss.Response = string(buf[0:n])

	// Actually check return code
	if n < 5 {
		err = errors.New("Server did not indicate support for STARTTLS")
	}
	if err == nil {
		var ret int
		ret, err = strconv.Atoi(ss.Response[0:3])
		if err != nil || ret < 200 || ret >= 300 {
			err = errors.New("Bad return code for STARTTLS")
		}
	}

	// Record everything no matter the result
	c.appendEvent(&ss, err)

	// Stop if we failed already
	if err != nil {
		return err
	}

	// Successful so far, attempt to do the actual handshake
	return c.TLSHandshake()
}

func (c *Conn) POP3StartTLSHandshake() error {
	ss := StartTLSEvent{Command: POP3_COMMAND}
	if err := c.sendStartTLSCommand(POP3_COMMAND); err != nil {
		c.appendEvent(&ss, err)
		return err
	}

	buf := make([]byte, 512)
	n, err := c.readPop3Response(buf)
	ss.Response = string(buf[0:n])
	if err == nil {
		if !strings.HasPrefix(ss.Response, "+") {
			err = errors.New("Server did not indicate support for STARTTLS")
		}
	}
	c.appendEvent(&ss, err)

	if err != nil {
		return err
	}
	return c.TLSHandshake()
}

func (c *Conn) IMAPStartTLSHandshake() error {
	ss := StartTLSEvent{Command: IMAP_COMMAND}
	if err := c.sendStartTLSCommand(IMAP_COMMAND); err != nil {
		c.appendEvent(&ss, err)
		return err
	}

	buf := make([]byte, 512)
	n, err := c.readImapStatusResponse(buf)
	ss.Response = string(buf[0:n])
	c.appendEvent(&ss, err)

	if err != nil {
		return err
	}
	return c.TLSHandshake()
}

func (c *Conn) readUntilRegex(res []byte, expr *regexp.Regexp) (int, error) {
	buf := res[0:]
	length := 0
	for finished := false; !finished; {
		n, err := c.getUnderlyingConn().Read(buf)
		length += n
		if err != nil {
			return length, err
		}
		if expr.Match(res[0:length]) {
			finished = true
		}
		if length == len(res) {
			return length, errors.New("Not enough buffer space")
		}
		buf = res[length:]
	}
	return length, nil
}

func (c *Conn) readSmtpResponse(res []byte) (int, error) {
	return c.readUntilRegex(res, smtpEndRegex)
}

func (c *Conn) SMTPBanner(b []byte) (int, error) {
	n, err := c.readSmtpResponse(b)
	mb := MailBannerEvent{}
	mb.Banner = string(b[0:n])
	c.appendEvent(&mb, err)
	return n, err
}

func (c *Conn) EHLO(domain string) error {
	cmd := []byte("EHLO " + domain + "\r\n")
	ee := EHLOEvent{}
	if _, err := c.getUnderlyingConn().Write(cmd); err != nil {
		c.appendEvent(&ee, err)
		return err
	}

	buf := make([]byte, 512)
	n, err := c.readSmtpResponse(buf)
	ee.Response = string(buf[0:n])
	c.appendEvent(&ee, err)
	return err
}

func (c *Conn) SMTPHelp() error {
	cmd := []byte("HELP\r\n")
	h := new(SMTPHelpEvent)
	if _, err := c.getUnderlyingConn().Write(cmd); err != nil {
		c.appendEvent(h, err)
		return err
	}
	buf := make([]byte, 512)
	n, err := c.readSmtpResponse(buf)
	h.Response = string(buf[0:n])
	c.appendEvent(h, err)
	return err
}

func (c *Conn) readPop3Response(res []byte) (int, error) {
	return c.readUntilRegex(res, pop3EndRegex)
}

func (c *Conn) POP3Banner(b []byte) (int, error) {
	n, err := c.readPop3Response(b)
	mb := MailBannerEvent{
		Banner: string(b[0:n]),
	}
	c.appendEvent(&mb, err)
	return n, err
}

func (c *Conn) readImapStatusResponse(res []byte) (int, error) {
	return c.readUntilRegex(res, imapStatusEndRegex)
}

func (c *Conn) IMAPBanner(b []byte) (int, error) {
	n, err := c.readImapStatusResponse(b)
	mb := MailBannerEvent{
		Banner: string(b[0:n]),
	}
	c.appendEvent(&mb, err)
	return n, err
}

func (c *Conn) CheckHeartbleed(b []byte) (int, error) {
	if !c.isTls {
		return 0, fmt.Errorf(
			"Must perform TLS handshake before sending Heartbleed probe to %s",
			c.RemoteAddr().String())
	}
	n, err := c.tlsConn.CheckHeartbleed(b)
	hb := c.tlsConn.GetHeartbleedLog()
	if err == ztls.HeartbleedError {
		err = nil
	}
	event := ConnectionEvent{
		Data:  &HeartbleedEvent{heartbleedLog: hb},
		Error: err,
	}
	c.operations = append(c.operations, event)
	return n, err
}

func (c *Conn) SendModbusEcho() (int, error) {
	req := ModbusRequest{
		Function: ModbusFunctionEncapsulatedInterface,
		Data: []byte{
			0x0E, // read device info
			0x01, // product code
			0x00, // object id, should always be 0 in initial request
		},
	}

	event := new(ModbusEvent)
	data, err := req.MarshalBinary()
	w := 0
	for w < len(data) {
		written, err := c.getUnderlyingConn().Write(data[w:]) // TODO verify write
		w += written
		if err != nil {
			c.appendEvent(event, err)
			return w, errors.New("Could not write modbus request")
		}
	}

	res, err := c.GetModbusResponse()
	event.Length = res.Length
	event.UnitID = res.UnitID
	event.Function = res.Function
	event.Response = res.Data
	event.ParseSelf()
	// make sure the whole thing gets appended to the operation log
	c.appendEvent(event, err)
	return w, err
}

func (c *Conn) States() []ConnectionEvent {
	return c.operations
}

func (c *Conn) appendEvent(data EventData, err error) {
	event := ConnectionEvent{
		Data:  data,
		Error: err,
	}
	c.operations = append(c.operations, event)
}
<|endoftext|>


<commit_before>package gosom

import (
	"io"
	"fmt"
	"math/rand"
	"encoding/json"
)

// SOM holds an instance of a self organizing map.
type SOM struct {
	Width int
	Height int
	Nodes []*Node
	CoolingFunction string
	DistanceFunction string
	NeighborhoodFunction string
}

// NewSOM creates and returns a new self organizing map.
func NewSOM(width, height int) *SOM {
	return &SOM{
		Width: width,
		Height: height,
		Nodes: make([]*Node, width * height),
		CoolingFunction: "linear",
		DistanceFunction: "euclidean",
		NeighborhoodFunction: "cone",
	}
}

func LoadSOMFromJSON(source io.Reader) (*SOM, error) {
	reader := json.NewDecoder(source)
	som := NewSOM(0, 0)

	err := reader.Decode(som)
	if err != nil {
		return nil, err
	}

	return som, nil
}

func (som *SOM) createNodes(dimensions int) {
	for i:=0; i<som.Height; i++ {
		for j:=0; j<som.Width; j++ {
			k := i * som.Width + j
			som.Nodes[k] = NewNode(j, i, dimensions)
		}
	}
}

func (som *SOM) InitializeWithRandomValues(matrix *Matrix) {
	som.createNodes(matrix.Columns)

	for _, node := range som.Nodes {
		for i:=0; i< matrix.Columns; i++ {
			r := (matrix.Maximums[i] - matrix.Minimums[i]) + matrix.Minimums[i]
			node.Weights[i] = r * rand.Float64()
		}
	}
}

func (som *SOM) InitializeWithDataPoints(matrix *Matrix) {
	som.createNodes(matrix.Columns)

	for _, node := range som.Nodes {
		copy(node.Weights, matrix.RandomRow())
	}
}

func (som *SOM) Closest(input []float64) *Node {
	nodes := make([]*Node, 0)

	// get initial distance
	t := som.Distance(input, som.Nodes[0].Weights)

	for _, node := range som.Nodes {
		// calculate distance
		d := som.Distance(input, node.Weights)

		if(d < t) {
			// save distance, clear array and add winner
			t = d
			nodes = append([]*Node{}, node)
		} else if(d == t) {
			// add winner
			nodes = append(nodes, node)
		}
	}

	if len(nodes) > 1 {
		// return random winner
		return nodes[rand.Intn(len(nodes))]
	}

	return nodes[0]
}

func (som *SOM) Neighbors(input []float64, K int) []*Node {
	nodes := make([]*Node, len(som.Nodes))
	copy(nodes, som.Nodes)

	SortNodes(nodes, func(n1, n2 *Node)(bool){
		d1 := som.Distance(input, n1.Weights)
		d2 := som.Distance(input, n2.Weights)

		return d1 < d2
	})

	neighbors := make([]*Node, 0, K)

	for i:=0; i<K; i++ {
		neighbors = append(neighbors, nodes[i])
	}

	return neighbors
}

func (som *SOM) Step(matrix *Matrix, step, steps int, initialLearningRate float64) {
	// calculate position
	progress := float64(step) / float64(steps)

	// calculate learning rate
	learningRate := initialLearningRate * som.CoolingFactor(progress)

	// calculate neighborhood radius
	initialRadius := float64(Max(som.Width, som.Height)) / 2.0
	radius := initialRadius * som.CoolingFactor(progress)

	// get random input
	input := matrix.RandomRow()

	// get closest node to input
	winningNode := som.Closest(input)

	for _, node := range som.Nodes {
		// calculate distance to winner
		distance := som.Distance(winningNode.Position, node.Position)

		// check inclusion in the radius (doubled to fit gaussian function)
		if(distance < radius * 2) {
			// calculate the influence
			i := som.NeighborhoodInfluenceFactor(distance / radius)

			// adjust node
			node.Adjust(input, i * learningRate)
		}
	}
}

func (som *SOM) Train(matrix *Matrix, steps int, initialLearningRate float64) {
	for step:=0; step<steps; step++ {
		som.Step(matrix, step, steps, initialLearningRate)
	}
}

func (som *SOM) Classify(input []float64) []float64 {
	o := make([]float64, som.Dimensions())
	copy(o, som.Closest(input).Weights)
	return o
}

func (som *SOM) Interpolate(input []float64, K int) []float64 {
	neighbors := som.Neighbors(input, K)
	total := make([]float64, som.Dimensions())

	// add up all values
	for i:=0; i<len(neighbors); i++ {
		for j:=0; j<som.Dimensions(); j++ {
			total[j] += neighbors[i].Weights[j]
		}
	}

	// calculate average
	for i:=0; i<som.Dimensions(); i++ {
		total[i] = total[i] / float64(K)
	}

	return total
}

func (som *SOM) WeightedInterpolate(input []float64, K int) []float64 {
	neighbors := som.Neighbors(input, K)
	neighborWeights := make([]float64, K)
	total := make([]float64, som.Dimensions())
	sumWeights := make([]float64, som.Dimensions())

	// calculate weights for neighbors
	radius := som.Distance(input, neighbors[K-1].Weights)
	for i, n := range neighbors {
		distance := som.Distance(input, n.Weights)
		neighborWeights[i] = som.NeighborhoodInfluenceFactor(distance / radius)
	}

	// add up all values
	for i:=0; i<len(neighbors); i++ {
		for j:=0; j<som.Dimensions(); j++ {
			total[j] += neighbors[i].Weights[j] * neighborWeights[i]
			sumWeights[j] += neighborWeights[i]
		}
	}

	// calculate average
	for i:=0; i<som.Dimensions(); i++ {
		total[i] = total[i] / sumWeights[i]
	}

	return total
}

// String returns a string matrix of all nodes and weights
func (som *SOM) String() string {
	s := ""

	for i:=0; i<som.Height; i++ {
		for j:=0; j<som.Width; j++ {
			k := i * som.Height + j
			s += fmt.Sprintf("%.2f ", som.Nodes[k].Weights)
		}

		s += "\n"
	}

	return s
}

func (som *SOM) CoolingFactor(progress float64) float64 {
	switch som.CoolingFunction {
	case "linear":
		return LinearCooling(progress)
	case "soft":
		return SoftCooling(progress)
	case "medium":
		return MediumCooling(progress)
	case "hard":
		return HardCooling(progress)
	}

	return 0.0
}

func (som *SOM) Distance(from, to []float64) float64 {
	switch som.DistanceFunction {
	case "euclidean":
		return EuclideanDistance(from, to)
	case "manhattan":
		return ManhattanDistance(from, to)
	}

	return 0.0
}

func (som *SOM) NeighborhoodInfluenceFactor(distance float64) float64 {
	switch som.NeighborhoodFunction {
	case "bubble":
		return BubbleNeighborhood(distance)
	case "cone":
		return ConeNeighborhood(distance)
	case "gaussian":
		return GaussianNeighborhood(distance)
	case "mexicanthat":
		return MexicanHatNeighborhood(distance)
	}

	return 0.0
}

func (som *SOM) Dimensions() int {
	return len(som.Nodes[0].Weights)
}

func (som *SOM) WeightMatrix() *Matrix {
	data := make([][]float64, len(som.Nodes))

	for i, node := range som.Nodes {
		data[i] = make([]float64, som.Dimensions())
		copy(data[i], node.Weights)
	}

	return NewMatrix(data)
}

func (som *SOM) Node(x, y int) *Node {
	return som.Nodes[y * som.Width + x]
}

func (som *SOM) SaveAsJSON(destination io.Writer) (error) {
	writer := json.NewEncoder(destination)
	return writer.Encode(som)
}
<commit_msg>variable renamed<commit_after>package gosom

import (
	"io"
	"fmt"
	"math/rand"
	"encoding/json"
)

// SOM holds an instance of a self organizing map.
type SOM struct {
	Width int
	Height int
	Nodes []*Node
	CoolingFunction string
	DistanceFunction string
	NeighborhoodFunction string
}

// NewSOM creates and returns a new self organizing map.
func NewSOM(width, height int) *SOM {
	return &SOM{
		Width: width,
		Height: height,
		Nodes: make([]*Node, width * height),
		CoolingFunction: "linear",
		DistanceFunction: "euclidean",
		NeighborhoodFunction: "cone",
	}
}

func LoadSOMFromJSON(source io.Reader) (*SOM, error) {
	reader := json.NewDecoder(source)
	som := NewSOM(0, 0)

	err := reader.Decode(som)
	if err != nil {
		return nil, err
	}

	return som, nil
}

func (som *SOM) createNodes(dimensions int) {
	for i:=0; i<som.Height; i++ {
		for j:=0; j<som.Width; j++ {
			k := i * som.Width + j
			som.Nodes[k] = NewNode(j, i, dimensions)
		}
	}
}

func (som *SOM) InitializeWithRandomValues(data *Matrix) {
	som.createNodes(data.Columns)

	for _, node := range som.Nodes {
		for i:=0; i< data.Columns; i++ {
			r := (data.Maximums[i] - data.Minimums[i]) + data.Minimums[i]
			node.Weights[i] = r * rand.Float64()
		}
	}
}

func (som *SOM) InitializeWithDataPoints(data *Matrix) {
	som.createNodes(data.Columns)

	for _, node := range som.Nodes {
		copy(node.Weights, data.RandomRow())
	}
}

func (som *SOM) Closest(input []float64) *Node {
	nodes := make([]*Node, 0)

	// get initial distance
	t := som.Distance(input, som.Nodes[0].Weights)

	for _, node := range som.Nodes {
		// calculate distance
		d := som.Distance(input, node.Weights)

		if(d < t) {
			// save distance, clear array and add winner
			t = d
			nodes = append([]*Node{}, node)
		} else if(d == t) {
			// add winner
			nodes = append(nodes, node)
		}
	}

	if len(nodes) > 1 {
		// return random winner
		return nodes[rand.Intn(len(nodes))]
	}

	return nodes[0]
}

func (som *SOM) Neighbors(input []float64, K int) []*Node {
	nodes := make([]*Node, len(som.Nodes))
	copy(nodes, som.Nodes)

	SortNodes(nodes, func(n1, n2 *Node)(bool){
		d1 := som.Distance(input, n1.Weights)
		d2 := som.Distance(input, n2.Weights)

		return d1 < d2
	})

	neighbors := make([]*Node, 0, K)

	for i:=0; i<K; i++ {
		neighbors = append(neighbors, nodes[i])
	}

	return neighbors
}

func (som *SOM) Step(data *Matrix, step, steps int, initialLearningRate float64) {
	// calculate position
	progress := float64(step) / float64(steps)

	// calculate learning rate
	learningRate := initialLearningRate * som.CoolingFactor(progress)

	// calculate neighborhood radius
	initialRadius := float64(Max(som.Width, som.Height)) / 2.0
	radius := initialRadius * som.CoolingFactor(progress)

	// get random input
	input := data.RandomRow()

	// get closest node to input
	winningNode := som.Closest(input)

	for _, node := range som.Nodes {
		// calculate distance to winner
		distance := som.Distance(winningNode.Position, node.Position)

		// check inclusion in the radius (doubled to fit gaussian function)
		if(distance < radius * 2) {
			// calculate the influence
			i := som.NeighborhoodInfluenceFactor(distance / radius)

			// adjust node
			node.Adjust(input, i * learningRate)
		}
	}
}

func (som *SOM) Train(data *Matrix, steps int, initialLearningRate float64) {
	for step:=0; step<steps; step++ {
		som.Step(data, step, steps, initialLearningRate)
	}
}

func (som *SOM) Classify(input []float64) []float64 {
	o := make([]float64, som.Dimensions())
	copy(o, som.Closest(input).Weights)
	return o
}

func (som *SOM) Interpolate(input []float64, K int) []float64 {
	neighbors := som.Neighbors(input, K)
	total := make([]float64, som.Dimensions())

	// add up all values
	for i:=0; i<len(neighbors); i++ {
		for j:=0; j<som.Dimensions(); j++ {
			total[j] += neighbors[i].Weights[j]
		}
	}

	// calculate average
	for i:=0; i<som.Dimensions(); i++ {
		total[i] = total[i] / float64(K)
	}

	return total
}

func (som *SOM) WeightedInterpolate(input []float64, K int) []float64 {
	neighbors := som.Neighbors(input, K)
	neighborWeights := make([]float64, K)
	total := make([]float64, som.Dimensions())
	sumWeights := make([]float64, som.Dimensions())

	// calculate weights for neighbors
	radius := som.Distance(input, neighbors[K-1].Weights)
	for i, n := range neighbors {
		distance := som.Distance(input, n.Weights)
		neighborWeights[i] = som.NeighborhoodInfluenceFactor(distance / radius)
	}

	// add up all values
	for i:=0; i<len(neighbors); i++ {
		for j:=0; j<som.Dimensions(); j++ {
			total[j] += neighbors[i].Weights[j] * neighborWeights[i]
			sumWeights[j] += neighborWeights[i]
		}
	}

	// calculate average
	for i:=0; i<som.Dimensions(); i++ {
		total[i] = total[i] / sumWeights[i]
	}

	return total
}

// String returns a string matrix of all nodes and weights
func (som *SOM) String() string {
	s := ""

	for i:=0; i<som.Height; i++ {
		for j:=0; j<som.Width; j++ {
			k := i * som.Height + j
			s += fmt.Sprintf("%.2f ", som.Nodes[k].Weights)
		}

		s += "\n"
	}

	return s
}

func (som *SOM) CoolingFactor(progress float64) float64 {
	switch som.CoolingFunction {
	case "linear":
		return LinearCooling(progress)
	case "soft":
		return SoftCooling(progress)
	case "medium":
		return MediumCooling(progress)
	case "hard":
		return HardCooling(progress)
	}

	return 0.0
}

func (som *SOM) Distance(from, to []float64) float64 {
	switch som.DistanceFunction {
	case "euclidean":
		return EuclideanDistance(from, to)
	case "manhattan":
		return ManhattanDistance(from, to)
	}

	return 0.0
}

func (som *SOM) NeighborhoodInfluenceFactor(distance float64) float64 {
	switch som.NeighborhoodFunction {
	case "bubble":
		return BubbleNeighborhood(distance)
	case "cone":
		return ConeNeighborhood(distance)
	case "gaussian":
		return GaussianNeighborhood(distance)
	case "mexicanthat":
		return MexicanHatNeighborhood(distance)
	}

	return 0.0
}

func (som *SOM) Dimensions() int {
	return len(som.Nodes[0].Weights)
}

func (som *SOM) WeightMatrix() *Matrix {
	data := make([][]float64, len(som.Nodes))

	for i, node := range som.Nodes {
		data[i] = make([]float64, som.Dimensions())
		copy(data[i], node.Weights)
	}

	return NewMatrix(data)
}

func (som *SOM) Node(x, y int) *Node {
	return som.Nodes[y * som.Width + x]
}

func (som *SOM) SaveAsJSON(destination io.Writer) (error) {
	writer := json.NewEncoder(destination)
	return writer.Encode(som)
}
<|endoftext|>


<commit_before>package rpio

import (
	"errors"
)

type SpiDev int

// SPI devices.
// Only SPI0 supported for now.
const (
	Spi0 SpiDev = iota
	Spi1        // aux
	Spi2        // aux
)

const (
	csReg     = 0
	fifoReg   = 1 // TX/RX FIFO
	clkDivReg = 2
)

var (
	SpiMapError = errors.New("SPI registers not mapped correctly - are you root?")
)

// Sets all pins of given SPI device to SPI mode
//  dev\pin | CE0 | CE1 | CE2 | SCLK | MOSI | MISO |
//  Spi0    |   7 |   8 |   - |    9 |   10 |   11 |
//  Spi1    |  16 |  17 |  18 |   19 |   20 |   21 |
//  Spi2    |  40 |  41 |  42 |   43 |   44 |   45 |
//
// It also resets SPI control register.
//
// Note that you should disable SPI interface in raspi-config first!
func SpiBegin(dev SpiDev) error {
	spiMem[csReg] = 0 // reset spi settings to default
	if spiMem[csReg] == 0 {
		// this should not read only zeroes after reset -> mem map failed
		return SpiMapError
	}

	for _, pin := range getSpiPins(dev) {
		pin.Mode(Spi)
	}

	clearSpiTxRxFifo()
	setSpiDiv(128)
	return nil
}

// Sets SPI pins of given device to default (Input) mode. See SpiBegin.
func SpiEnd(dev SpiDev) {
	var pins = getSpiPins(dev)
	for _, pin := range pins {
		pin.Mode(Input)
	}
}

// Set (maximal) speed [Hz] of SPI clock.
// Param speed may be as big as 125MHz in theory, but
// only values up to 31.25MHz are considered relayable.
func SpiSpeed(speed int) {
	const baseFreq = 250 * 1000000
	cdiv := uint32(baseFreq / speed)
	setSpiDiv(cdiv)
}

// Select chip, one of 0, 1, 2
// for selecting slave on CE0, CE1, or CE2 pin
func SpiChipSelect(chip int) {
	const csMask = 3 // chip select has 2 bits

	cs := uint32(chip & csMask)

	spiMem[csReg] = spiMem[csReg]&^csMask | cs
}

// SpiTransmit takes one or more bytes and send them to slave.
//
// Data received from slave are ignored.
// Use spread operator to send slice of bytes.
func SpiTransmit(data ...byte) {
	SpiExchange(append(data[:0:0], data...)) // clone data because it will be rewriten by received bytes
}

// SpiReceive receives n bytes from slave.
//
// Note that n zeroed bytes are send to slave as side effect.
func SpiReceive(n int) []byte {
	data := make([]byte, n, n)
	SpiExchange(data)
	return data
}

// Transmit all bytes in data to slave
// and simultaneously receives bytes from slave to data.
//
// If you want to only send or only receive, use SpiTransmit/SpiReceive
func SpiExchange(data []byte) {
	const ta = 1 << 7   // transfer active
	const txd = 1 << 18 // tx fifo can accept data
	const rxd = 1 << 17 // rx fifo contains data
	const done = 1 << 16

	clearSpiTxRxFifo()

	// set TA = 1
	spiMem[csReg] |= ta

	for i := range data {
		// wait for TXD
		for spiMem[csReg]&txd == 0 {
		}
		// write bytes to SPI_FIFO
		spiMem[fifoReg] = uint32(data[i])

		// wait for RXD
		for spiMem[csReg]&rxd == 0 {
		}
		// read bytes from SPI_FIFO
		data[i] = byte(spiMem[fifoReg])
	}

	// wait for DONE
	for spiMem[csReg]&done == 0 {
	}

	// Set TA = 0
	spiMem[csReg] &^= ta
}

// set spi clock divider value
func setSpiDiv(div uint32) {
	const divMask = 1<<16 - 1 - 1 // cdiv have 16 bits and must be odd (for some reason)
	spiMem[clkDivReg] = div & divMask
}

// clear both FIFOs
func clearSpiTxRxFifo() {
	const clearTxRx = 1<<5 | 1<<4
	spiMem[csReg] |= clearTxRx
}

func getSpiPins(dev SpiDev) []Pin {
	switch dev {
	case Spi0:
		return []Pin{7, 8, 9, 10, 11}
		// ommit 35, 36, 37, 38, 39 - only one set of SPI0 can be set in Spi mode at a time
	case Spi1:
		return []Pin{16, 17, 18, 19, 20, 21}
	case Spi2:
		return []Pin{40, 41, 42, 43, 44, 45}
	default:
		return []Pin{}
	}
}
<commit_msg>Add clockmode and cspolarity SPI settings<commit_after>package rpio

import (
	"errors"
)

type SpiDev int

// SPI devices.
// Only SPI0 supported for now.
const (
	Spi0 SpiDev = iota
	Spi1        // aux
	Spi2        // aux
)

const (
	csReg     = 0
	fifoReg   = 1 // TX/RX FIFO
	clkDivReg = 2
)

var (
	SpiMapError = errors.New("SPI registers not mapped correctly - are you root?")
)

// Sets all pins of given SPI device to SPI mode
//  dev\pin | CE0 | CE1 | CE2 | SCLK | MOSI | MISO |
//  Spi0    |   7 |   8 |   - |    9 |   10 |   11 |
//  Spi1    |  16 |  17 |  18 |   19 |   20 |   21 |
//  Spi2    |  40 |  41 |  42 |   43 |   44 |   45 |
//
// It also resets SPI control register.
//
// Note that you should disable SPI interface in raspi-config first!
func SpiBegin(dev SpiDev) error {
	spiMem[csReg] = 0 // reset spi settings to default
	if spiMem[csReg] == 0 {
		// this should not read only zeroes after reset -> mem map failed
		return SpiMapError
	}

	for _, pin := range getSpiPins(dev) {
		pin.Mode(Spi)
	}

	clearSpiTxRxFifo()
	setSpiDiv(128)
	return nil
}

// Sets SPI pins of given device to default (Input) mode. See SpiBegin.
func SpiEnd(dev SpiDev) {
	var pins = getSpiPins(dev)
	for _, pin := range pins {
		pin.Mode(Input)
	}
}

// Set (maximal) speed [Hz] of SPI clock.
// Param speed may be as big as 125MHz in theory, but
// only values up to 31.25MHz are considered relayable.
func SpiSpeed(speed int) {
	const baseFreq = 250 * 1000000
	cdiv := uint32(baseFreq / speed)
	setSpiDiv(cdiv)
}

// Select chip, one of 0, 1, 2
// for selecting slave on CE0, CE1, or CE2 pin
func SpiChipSelect(chip uint8) {
	const csMask = 3 // chip select has 2 bits

	cs := uint32(chip & csMask)

	spiMem[csReg] = spiMem[csReg]&^csMask | cs
}

// Sets polarity (0/1) of active chip select
// default active=0
func SpiChipSelectPolarity(chip uint8, polarity uint8) {
	if chip > 2 {
		return
	}
	cspol := uint32(1 << (21 + chip)) // bit 21, 22 or 23 depending on chip

	if polarity == 0 { // chip select is active low
		spiMem[csReg] &^= cspol
	} else { // chip select is active hight
		spiMem[csReg] |= cspol
	}
}

// Set polarity (0/1) and phase (0/1) of spi clock
// default polarity=0; phase=0
func SpiMode(polarity uint8, phase uint8) {
	const cpol = 1 << 3
	const cpha = 1 << 2

	if polarity == 0 { // Rest state of clock = low
		spiMem[csReg] &^= cpol
	} else { // Rest state of clock = high
		spiMem[csReg] |= cpol
	}

	if phase == 0 { // First SCLK transition at middle of data bit
		spiMem[csReg] &^= cpha
	} else { // First SCLK transition at beginning of data bit
		spiMem[csReg] |= cpha
	}
}

// SpiTransmit takes one or more bytes and send them to slave.
//
// Data received from slave are ignored.
// Use spread operator to send slice of bytes.
func SpiTransmit(data ...byte) {
	SpiExchange(append(data[:0:0], data...)) // clone data because it will be rewriten by received bytes
}

// SpiReceive receives n bytes from slave.
//
// Note that n zeroed bytes are send to slave as side effect.
func SpiReceive(n int) []byte {
	data := make([]byte, n, n)
	SpiExchange(data)
	return data
}

// Transmit all bytes in data to slave
// and simultaneously receives bytes from slave to data.
//
// If you want to only send or only receive, use SpiTransmit/SpiReceive
func SpiExchange(data []byte) {
	const ta = 1 << 7   // transfer active
	const txd = 1 << 18 // tx fifo can accept data
	const rxd = 1 << 17 // rx fifo contains data
	const done = 1 << 16

	clearSpiTxRxFifo()

	// set TA = 1
	spiMem[csReg] |= ta

	for i := range data {
		// wait for TXD
		for spiMem[csReg]&txd == 0 {
		}
		// write bytes to SPI_FIFO
		spiMem[fifoReg] = uint32(data[i])

		// wait for RXD
		for spiMem[csReg]&rxd == 0 {
		}
		// read bytes from SPI_FIFO
		data[i] = byte(spiMem[fifoReg])
	}

	// wait for DONE
	for spiMem[csReg]&done == 0 {
	}

	// Set TA = 0
	spiMem[csReg] &^= ta
}

// set spi clock divider value
func setSpiDiv(div uint32) {
	const divMask = 1<<16 - 1 - 1 // cdiv have 16 bits and must be odd (for some reason)
	spiMem[clkDivReg] = div & divMask
}

// clear both FIFOs
func clearSpiTxRxFifo() {
	const clearTxRx = 1<<5 | 1<<4
	spiMem[csReg] |= clearTxRx
}

func getSpiPins(dev SpiDev) []Pin {
	switch dev {
	case Spi0:
		return []Pin{7, 8, 9, 10, 11}
		// ommit 35, 36, 37, 38, 39 - only one set of SPI0 can be set in Spi mode at a time
	case Spi1:
		return []Pin{16, 17, 18, 19, 20, 21}
	case Spi2:
		return []Pin{40, 41, 42, 43, 44, 45}
	default:
		return []Pin{}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
)

type sse struct {
	events       chan []byte
	addClient    chan chan []byte
	removeClient chan chan []byte
	clients      map[chan []byte]bool
}

func newSSE() *sse {
	return &sse{
		events:       make(chan []byte, 0),
		addClient:    make(chan chan []byte, 0),
		removeClient: make(chan chan []byte, 0),
		clients:      make(map[chan []byte]bool, 0),
	}
}

func (sse *sse) Loop() {
	for {
		select {
		case s := <-sse.addClient:
			sse.clients[s] = true
			log.Println("Added sse client.", len(sse.clients))
		case s := <-sse.removeClient:
			delete(sse.clients, s)
			log.Println("Removed sse client.", len(sse.clients))
		case event := <-sse.events:
			for client := range sse.clients {
				client <- event
			}
		}
	}
}

var ssecounter = 0

func (sse *sse) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/event-stream")
	w.Header().Add("Cache-Control", "no-cache")
	w.Header().Add("Connection", "keep-alive")
	w.(http.Flusher).Flush()

	ssecounter++
	id := ssecounter
	header := r.Header["User-Agent"]

	events := make(chan []byte)
	sse.addClient <- events

	defer func() {
		sse.removeClient <- events
	}()

	go func() {
		<-w.(http.CloseNotifier).CloseNotify()
		sse.removeClient <- events
		close(events)
	}()

	newline := []byte("\n")
	prefix := []byte("\ndata: ")
	for event := range events {
		// When we see a newline we need to add the prefix again.
		event = bytes.Replace(event, newline, prefix, -1)
		if _, err := fmt.Fprintf(w, "data: %s\n\n", event); err != nil {
			log.Println("Error writing to SSE", id, header, err)
			break
		}
		w.(http.Flusher).Flush()
	}
}
<commit_msg>Making the sse fail continue instead of break<commit_after>package main

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
)

type sse struct {
	events       chan []byte
	addClient    chan chan []byte
	removeClient chan chan []byte
	clients      map[chan []byte]bool
}

func newSSE() *sse {
	return &sse{
		events:       make(chan []byte, 0),
		addClient:    make(chan chan []byte, 0),
		removeClient: make(chan chan []byte, 0),
		clients:      make(map[chan []byte]bool, 0),
	}
}

func (sse *sse) Loop() {
	for {
		select {
		case s := <-sse.addClient:
			sse.clients[s] = true
			log.Println("Added sse client.", len(sse.clients))
		case s := <-sse.removeClient:
			delete(sse.clients, s)
			log.Println("Removed sse client.", len(sse.clients))
		case event := <-sse.events:
			for client := range sse.clients {
				client <- event
			}
		}
	}
}

var ssecounter = 0

func (sse *sse) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/event-stream")
	w.Header().Add("Cache-Control", "no-cache")
	w.Header().Add("Connection", "keep-alive")
	w.(http.Flusher).Flush()

	ssecounter++
	id := ssecounter
	header := r.Header["User-Agent"]

	events := make(chan []byte)
	sse.addClient <- events

	defer func() {
		sse.removeClient <- events
	}()

	go func() {
		<-w.(http.CloseNotifier).CloseNotify()
		sse.removeClient <- events
		close(events)
	}()

	newline := []byte("\n")
	prefix := []byte("\ndata: ")
	for event := range events {
		// When we see a newline we need to add the prefix again.
		event = bytes.Replace(event, newline, prefix, -1)
		if _, err := fmt.Fprintf(w, "data: %s\n\n", event); err != nil {
			log.Println("Error writing to SSE", id, header, err)
			continue
		}
		w.(http.Flusher).Flush()
	}
}
<|endoftext|>


<commit_before>package model

import (
	"database/sql"
	"errors"
	"strconv"
	"strings"

	_ "github.com/go-sql-driver/mysql"
)

type DB struct {
	db *sql.DB
}

func (db *DB) Connect() error {
	var err error
	db.db, err = sql.Open("mysql", "root:password@/kyuko")
	return err
}

func (db *DB) Close() error {
	err := db.db.Close()
	return err
}

func (db *DB) Insert(k KyukoData) (sql.Result, error) {
	result, err := db.db.Exec("INSERT INTO `kyuko_data` VALUES(?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `reason`=?;", 0, k.Place, k.Weekday, k.Period, k.Day, k.ClassName, k.Instructor, k.Reason, k.Reason)
	return result, err
}

func (db *DB) InsertCanceledClass(c CanceledClass) (sql.Result, error) {
	sql := "INSERT INTO `canceled_class` (canceled, place, week, period, year, season, class_name, instructor) VALUES(?, ?, ?, ?, ?, ?, ?, ?);"

	result, err := db.db.Exec(sql, c.Canceled, c.Place, c.Weekday, c.Period, c.Year, c.Season, c.ClassName, c.Instructor)
	return result, err
}

func (db *DB) InsertDay(d Day) (sql.Result, error) {
	sql := "INSERT INTO `day` (canceled_class_id, day) VALUES(?, ?);"
	result, err := db.db.Exec(sql, d.CanceledClassID, d.Date)
	return result, err
}

func (db *DB) InsertReason(r Reason) (sql.Result, error) {
	sql := "INSERT INTO `reason` (canceled_class_id, reason) VALUES(?, ?);"
	result, err := db.db.Exec(sql, r.CanceledClassID, r.Reason)
	return result, err
}

func ScanAll(rows *sql.Rows) ([]KyukoData, error) {
	kyukoData := []KyukoData{}
	var err error
	for rows.Next() {
		var k KyukoData
		if err = rows.Scan(&k.ID, &k.Place, &k.Weekday, &k.Period, &k.Day, &k.ClassName, &k.Instructor, &k.Reason); err != nil {
			return kyukoData, err
		}
		kyukoData = append(kyukoData, k)
	}
	return kyukoData, err
}

func ScanCanceledClass(rows *sql.Rows) ([]CanceledClass, error) {
	canceledClass := []CanceledClass{}
	var err error
	for rows.Next() {
		var c CanceledClass
		if err = rows.Scan(&c.ID, &c.Canceled, &c.Place, &c.Weekday, &c.Period, &c.Year, &c.Season, &c.ClassName, &c.Instructor); err != nil {
			return canceledClass, err
		}
		canceledClass = append(canceledClass, c)
	}
	return canceledClass, err
}

func ScanDay(rows *sql.Rows) ([]Day, error) {
	days := []Day{}
	var err error
	for rows.Next() {
		var d Day
		if err = rows.Scan(&d.ID, &d.CanceledClassID, &d.Date); err != nil {
			return days, err
		}
		days = append(days, d)
	}
	return days, err
}

func ScanReason(rows *sql.Rows) ([]Reason, error) {
	reasons := []Reason{}
	var err error
	for rows.Next() {
		var r Reason
		if err = rows.Scan(&r.ID, &r.CanceledClassID, &r.Reason); err != nil {
			return reasons, err
		}
		reasons = append(reasons, r)
	}
	return reasons, err
}

func (db *DB) SelectAll() ([]KyukoData, error) {
	kyukoData := []KyukoData{}
	rows, err := db.db.Query("select * from kyuko_data where id in(select min(id) from kyuko_data group by class_name, day);")
	if err != nil {
		return kyukoData, err
	}
	defer rows.Close()
	kyukoData, err = ScanAll(rows)
	if err != nil {
		return kyukoData, err
	}
	return kyukoData, err
}

func (db *DB) SelectReaonFromCanceledID(canceledID int) ([]Reason, error) {
	reasons := []Reason{}
	sql := "SELECT * FROM reason WHERE canceled_id=?"
	rows, err := db.db.Query(sql, canceledID)
	if err != nil {
		return reasons, err
	}
	defer rows.Close()
	reasons, err = ScanReason(rows)
	if err != nil {
		return reasons, err
	}

	return reasons, nil
}

// DBからIDを取得します。
// 存在しない場合は-1を返します
// 一意に定まらない場合はerrorを返します
func (db *DB) ShowCanceledClassID(c CanceledClass) (int, error) {
	sql := "select * from canceled_class where class_name=? and year=? and season = ?"
	rows, err := db.db.Query(sql, c.ClassName, c.Year, c.Season)
	if err != nil {
		return -1, err
	}
	defer rows.Close()
	canceledclass := []CanceledClass{}
	canceledclass, err = ScanCanceledClass(rows)
	if err != nil {
		return -1, err
	}

	if len(canceledclass) != 1 && len(canceledclass) != 0 {
		return -1, errors.New("IDが一意に定まりませんでした")
	} else if len(canceledclass) == 0 {
		return -1, nil
	}
	return canceledclass[0].ID, nil
}

func (db *DB) ShowCanceled(id int) (int, error) {
	sql := "select * from canceled_class where id=?;"
	rows, err := db.db.Query(sql, id)
	if err != nil {
		return -1, err
	}
	defer rows.Close()
	canceledclass := []CanceledClass{}
	canceledclass, err = ScanCanceledClass(rows)
	if err != nil {
		return -1, err
	}
	if len(canceledclass) != 1 && len(canceledclass) != 0 {
		return -1, errors.New("IDが一意に定まりませんでした")
	} else if len(canceledclass) == 0 {
		return -1, nil
	}
	return canceledclass[0].Canceled, nil
}

// Dayテーブルをみて今日の日付があるのか確認
func (db *DB) IsExistToday(id int, date string) (bool, error) {
	sql := "select * from day where canceled_class_id=?"
	rows, err := db.db.Query(sql, id)
	if err != nil {
		return false, err
	}
	defer rows.Close()
	days := []Day{}
	days, err = ScanDay(rows)
	if err != nil {
		return false, err
	}

	//dayが今日の日付があるのか
	for _, day := range days {
		if day.Date == date {
			return true, nil
		}
	}

	return false, nil
}

func (db *DB) DeleteWhereDayAndClassName(day, className string) (sql.Result, error) {
	result, err := db.db.Exec("delete from kyuko_data where day = ? and class_name = ?", day, className)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) AddCanceled(id int) (sql.Result, error) {
	sql := "UPDATE canceled_class SET canceled = canceled+1 WHERE id = ?;"
	result, err := db.db.Exec(sql, id)
	if err != nil {
		return result, err
	}
	return result, err
}

func KyukoToCanceled(k KyukoData) (CanceledClass, error) {
	season, err := getSeason(k.Day)
	if err != nil {
		return CanceledClass{}, err
	}

	year, err := getYear(k.Day)
	if err != nil {
		return CanceledClass{}, err
	}

	return CanceledClass{
		Place:      k.Place,
		Weekday:    k.Weekday,
		Period:     k.Period,
		ClassName:  k.ClassName,
		Instructor: k.Instructor,
		Season:     season,
		Year:       year,
	}, nil
}

func getYear(day string) (int, error) {
	strYear := strings.Split(day, "/")[0]
	year, _ := strconv.Atoi(strYear)
	return year, nil
}

func getSeason(day string) (string, error) {
	strMonth := ""
	if strings.Contains(day, "-") {
		strMonth = strings.Split(day, "-")[1]
	} else {
		strMonth = strings.Split(day, "/")[1]
	}

	month, _ := strconv.Atoi(strMonth)

	if month >= 3 && month <= 8 {
		return "spring", nil
	} else if (month >= 9 && month <= 12) || (month >= 1 && month <= 2) {
		return "autumn", nil
	}

	return "", errors.New("Season are not uniquely determined")
}

func (db *DB) DeleteCanceled(id int) (sql.Result, error) {
	sql := "DELETE FROM canceled_class WHERE id = ?;"
	result, err := db.db.Exec(sql, id)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) deleteReason(id int) (sql.Result, error) {
	sql := "DELETE FROM reason WHERE id = ?;"
	result, err := db.db.Exec(sql, id)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) DeleteReasonWhere(canceledID int, reason string) (sql.Result, error) {
	sql := "DELETE FROM reason WHERE canceled_class_id = ? AND reason = ?;"
	result, err := db.db.Exec(sql, canceledID, reason)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) deleteDay(id int) (sql.Result, error) {
	sql := "DELETE FROM day WHERE id = ?;"
	result, err := db.db.Exec(sql, id)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) DeleteDayWhere(canceledID int, day string) (sql.Result, error) {
	sql := "DELETE FROM day WHERE canceled_class_id = ? AND day = ?;"
	result, err := db.db.Exec(sql, canceledID, day)
	if err != nil {
		return result, err
	}
	return result, err
}
<commit_msg>name resolution<commit_after>package model

import (
	"database/sql"
	"errors"
	"strconv"
	"strings"

	_ "github.com/go-sql-driver/mysql"
)

type DB struct {
	db *sql.DB
}

func (db *DB) Connect() error {
	var err error
	db.db, err = sql.Open("mysql", "root:password@tcp(kyuko-db:3306)/kyuko")
	return err
}

func (db *DB) Close() error {
	err := db.db.Close()
	return err
}

func (db *DB) Insert(k KyukoData) (sql.Result, error) {
	result, err := db.db.Exec("INSERT INTO `kyuko_data` VALUES(?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE `reason`=?;", 0, k.Place, k.Weekday, k.Period, k.Day, k.ClassName, k.Instructor, k.Reason, k.Reason)
	return result, err
}

func (db *DB) InsertCanceledClass(c CanceledClass) (sql.Result, error) {
	sql := "INSERT INTO `canceled_class` (canceled, place, week, period, year, season, class_name, instructor) VALUES(?, ?, ?, ?, ?, ?, ?, ?);"

	result, err := db.db.Exec(sql, c.Canceled, c.Place, c.Weekday, c.Period, c.Year, c.Season, c.ClassName, c.Instructor)
	return result, err
}

func (db *DB) InsertDay(d Day) (sql.Result, error) {
	sql := "INSERT INTO `day` (canceled_class_id, day) VALUES(?, ?);"
	result, err := db.db.Exec(sql, d.CanceledClassID, d.Date)
	return result, err
}

func (db *DB) InsertReason(r Reason) (sql.Result, error) {
	sql := "INSERT INTO `reason` (canceled_class_id, reason) VALUES(?, ?);"
	result, err := db.db.Exec(sql, r.CanceledClassID, r.Reason)
	return result, err
}

func ScanAll(rows *sql.Rows) ([]KyukoData, error) {
	kyukoData := []KyukoData{}
	var err error
	for rows.Next() {
		var k KyukoData
		if err = rows.Scan(&k.ID, &k.Place, &k.Weekday, &k.Period, &k.Day, &k.ClassName, &k.Instructor, &k.Reason); err != nil {
			return kyukoData, err
		}
		kyukoData = append(kyukoData, k)
	}
	return kyukoData, err
}

func ScanCanceledClass(rows *sql.Rows) ([]CanceledClass, error) {
	canceledClass := []CanceledClass{}
	var err error
	for rows.Next() {
		var c CanceledClass
		if err = rows.Scan(&c.ID, &c.Canceled, &c.Place, &c.Weekday, &c.Period, &c.Year, &c.Season, &c.ClassName, &c.Instructor); err != nil {
			return canceledClass, err
		}
		canceledClass = append(canceledClass, c)
	}
	return canceledClass, err
}

func ScanDay(rows *sql.Rows) ([]Day, error) {
	days := []Day{}
	var err error
	for rows.Next() {
		var d Day
		if err = rows.Scan(&d.ID, &d.CanceledClassID, &d.Date); err != nil {
			return days, err
		}
		days = append(days, d)
	}
	return days, err
}

func ScanReason(rows *sql.Rows) ([]Reason, error) {
	reasons := []Reason{}
	var err error
	for rows.Next() {
		var r Reason
		if err = rows.Scan(&r.ID, &r.CanceledClassID, &r.Reason); err != nil {
			return reasons, err
		}
		reasons = append(reasons, r)
	}
	return reasons, err
}

func (db *DB) SelectAll() ([]KyukoData, error) {
	kyukoData := []KyukoData{}
	rows, err := db.db.Query("select * from kyuko_data where id in(select min(id) from kyuko_data group by class_name, day);")
	if err != nil {
		return kyukoData, err
	}
	defer rows.Close()
	kyukoData, err = ScanAll(rows)
	if err != nil {
		return kyukoData, err
	}
	return kyukoData, err
}

func (db *DB) SelectReaonFromCanceledID(canceledID int) ([]Reason, error) {
	reasons := []Reason{}
	sql := "SELECT * FROM reason WHERE canceled_id=?"
	rows, err := db.db.Query(sql, canceledID)
	if err != nil {
		return reasons, err
	}
	defer rows.Close()
	reasons, err = ScanReason(rows)
	if err != nil {
		return reasons, err
	}

	return reasons, nil
}

// DBからIDを取得します。
// 存在しない場合は-1を返します
// 一意に定まらない場合はerrorを返します
func (db *DB) ShowCanceledClassID(c CanceledClass) (int, error) {
	sql := "select * from canceled_class where class_name=? and year=? and season = ?"
	rows, err := db.db.Query(sql, c.ClassName, c.Year, c.Season)
	if err != nil {
		return -1, err
	}
	defer rows.Close()
	canceledclass := []CanceledClass{}
	canceledclass, err = ScanCanceledClass(rows)
	if err != nil {
		return -1, err
	}

	if len(canceledclass) != 1 && len(canceledclass) != 0 {
		return -1, errors.New("IDが一意に定まりませんでした")
	} else if len(canceledclass) == 0 {
		return -1, nil
	}
	return canceledclass[0].ID, nil
}

func (db *DB) ShowCanceled(id int) (int, error) {
	sql := "select * from canceled_class where id=?;"
	rows, err := db.db.Query(sql, id)
	if err != nil {
		return -1, err
	}
	defer rows.Close()
	canceledclass := []CanceledClass{}
	canceledclass, err = ScanCanceledClass(rows)
	if err != nil {
		return -1, err
	}
	if len(canceledclass) != 1 && len(canceledclass) != 0 {
		return -1, errors.New("IDが一意に定まりませんでした")
	} else if len(canceledclass) == 0 {
		return -1, nil
	}
	return canceledclass[0].Canceled, nil
}

// Dayテーブルをみて今日の日付があるのか確認
func (db *DB) IsExistToday(id int, date string) (bool, error) {
	sql := "select * from day where canceled_class_id=?"
	rows, err := db.db.Query(sql, id)
	if err != nil {
		return false, err
	}
	defer rows.Close()
	days := []Day{}
	days, err = ScanDay(rows)
	if err != nil {
		return false, err
	}

	//dayが今日の日付があるのか
	for _, day := range days {
		if day.Date == date {
			return true, nil
		}
	}

	return false, nil
}

func (db *DB) DeleteWhereDayAndClassName(day, className string) (sql.Result, error) {
	result, err := db.db.Exec("delete from kyuko_data where day = ? and class_name = ?", day, className)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) AddCanceled(id int) (sql.Result, error) {
	sql := "UPDATE canceled_class SET canceled = canceled+1 WHERE id = ?;"
	result, err := db.db.Exec(sql, id)
	if err != nil {
		return result, err
	}
	return result, err
}

func KyukoToCanceled(k KyukoData) (CanceledClass, error) {
	season, err := getSeason(k.Day)
	if err != nil {
		return CanceledClass{}, err
	}

	year, err := getYear(k.Day)
	if err != nil {
		return CanceledClass{}, err
	}

	return CanceledClass{
		Place:      k.Place,
		Weekday:    k.Weekday,
		Period:     k.Period,
		ClassName:  k.ClassName,
		Instructor: k.Instructor,
		Season:     season,
		Year:       year,
	}, nil
}

func getYear(day string) (int, error) {
	strYear := strings.Split(day, "/")[0]
	year, _ := strconv.Atoi(strYear)
	return year, nil
}

func getSeason(day string) (string, error) {
	strMonth := ""
	if strings.Contains(day, "-") {
		strMonth = strings.Split(day, "-")[1]
	} else {
		strMonth = strings.Split(day, "/")[1]
	}

	month, _ := strconv.Atoi(strMonth)

	if month >= 3 && month <= 8 {
		return "spring", nil
	} else if (month >= 9 && month <= 12) || (month >= 1 && month <= 2) {
		return "autumn", nil
	}

	return "", errors.New("Season are not uniquely determined")
}

func (db *DB) DeleteCanceled(id int) (sql.Result, error) {
	sql := "DELETE FROM canceled_class WHERE id = ?;"
	result, err := db.db.Exec(sql, id)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) deleteReason(id int) (sql.Result, error) {
	sql := "DELETE FROM reason WHERE id = ?;"
	result, err := db.db.Exec(sql, id)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) DeleteReasonWhere(canceledID int, reason string) (sql.Result, error) {
	sql := "DELETE FROM reason WHERE canceled_class_id = ? AND reason = ?;"
	result, err := db.db.Exec(sql, canceledID, reason)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) deleteDay(id int) (sql.Result, error) {
	sql := "DELETE FROM day WHERE id = ?;"
	result, err := db.db.Exec(sql, id)
	if err != nil {
		return result, err
	}
	return result, err
}

func (db *DB) DeleteDayWhere(canceledID int, day string) (sql.Result, error) {
	sql := "DELETE FROM day WHERE canceled_class_id = ? AND day = ?;"
	result, err := db.db.Exec(sql, canceledID, day)
	if err != nil {
		return result, err
	}
	return result, err
}
<|endoftext|>


<commit_before>package models

import (
	"github.com/stainberg/koalart"
	"io"
	"mirbase"
	"net/http"
	"utils"
)

type LinkController struct {
	koala.Controller
}

func (k *LinkController) URLMapping() {
	k.Mapping(koala.GET, k.Get)
}

func (c *LinkController) Get() {
	if !utils.CheckToken(c.Ctx.Request.Header.Get("token")) {
		c.Ctx.Writer.WriteHeader(http.StatusForbidden)
		io.WriteString(c.Ctx.Writer, "illegal token")
		return
	}
	c.Ctx.Writer.WriteHeader(http.StatusOK)
	url := c.Ctx.Query.Get("url")
	link := mirbase.SaveShortLink(url)
	io.WriteString(c.Ctx.Writer, c.Ctx.Request.Host + "/" + link)
}<commit_msg>add http  prefix.<commit_after>package models

import (
	"github.com/stainberg/koalart"
	"io"
	"mirbase"
	"net/http"
	"utils"
)

type LinkController struct {
	koala.Controller
}

func (k *LinkController) URLMapping() {
	k.Mapping(koala.GET, k.Get)
}

func (c *LinkController) Get() {
	if !utils.CheckToken(c.Ctx.Request.Header.Get("token")) {
		c.Ctx.Writer.WriteHeader(http.StatusForbidden)
		io.WriteString(c.Ctx.Writer, "illegal token")
		return
	}
	c.Ctx.Writer.WriteHeader(http.StatusOK)
	url := c.Ctx.Query.Get("url")
	link := mirbase.SaveShortLink(url)
	io.WriteString(c.Ctx.Writer, "http://" + c.Ctx.Request.Host + "/" + link)
}<|endoftext|>


<commit_before>package crawler

import (
	"bytes"
	"fmt"
	"log"
	"text/template"
	"time"

	"github.com/janicduplessis/resultscrawler/pkg/api"
	"github.com/janicduplessis/resultscrawler/pkg/store/crawlerconfig"
	"github.com/janicduplessis/resultscrawler/pkg/store/results"
	"github.com/janicduplessis/resultscrawler/pkg/store/user"
	"github.com/janicduplessis/resultscrawler/pkg/tools"
)

const (
	// Time between checks to see if a user needs an update in seconds
	checkInterval time.Duration = 30 * time.Second
	// Time between updates for each user in minutes
	updateInterval time.Duration = 1 * time.Minute
)

var msgTemplatePath = "msgtemplate.html"
var msgTemplate *template.Template

type crawlerUser struct {
	ID      string
	Classes []api.Class
	Nip     string
	Code    string
	Name    string
	Email   string
}

// SchedulerConfig initializes the scheduler.
type SchedulerConfig struct {
	Crawlers           []*Crawler
	UserStore          user.Store
	CrawlerConfigStore crawlerconfig.Store
	UserResultsStore   results.Store
	Sender             tools.Sender
}

// Scheduler handles scheduling crawler runs for every user.
type Scheduler struct {
	Crawlers           []*Crawler
	UserStore          user.Store
	CrawlerConfigStore crawlerconfig.Store
	UserResultsStore   results.Store
	Sender             tools.Sender

	queueCh chan *crawlerUser
	doneCh  chan bool
}

// NewScheduler creates a new scuduler object.
func NewScheduler(config *SchedulerConfig) *Scheduler {
	if msgTemplate == nil {
		msgTemplate = template.Must(template.New("msgtemplate.html").ParseFiles(msgTemplatePath))
	}

	queueCh := make(chan *crawlerUser)
	doneCh := make(chan bool)

	return &Scheduler{
		config.Crawlers,
		config.UserStore,
		config.CrawlerConfigStore,
		config.UserResultsStore,
		config.Sender,

		queueCh,
		doneCh,
	}
}

// Start starts the scheduler
func (s *Scheduler) Start() {
	for _, crawler := range s.Crawlers {
		go s.crawlerLoop(crawler)
	}

	s.mainLoop()
}

// Queue tells the scheduler do a run for a user
func (s *Scheduler) Queue(user *api.User) {
	// Get the user current results
	results, err := s.UserResultsStore.GetResults(user.ID)
	if err != nil {
		log.Println(err)
		return
	}

	// Check la update time.
	if time.Now().Sub(results.LastUpdate) < updateInterval {
		return
	}

	// Get crawler config.
	crawlerConfig, err := s.CrawlerConfigStore.GetCrawlerConfig(user.ID)
	if err != nil {
		log.Println(err)
		return
	}

	s.queueCh <- &crawlerUser{
		ID:      user.ID,
		Classes: results.Classes,
		Code:    crawlerConfig.Code,
		Nip:     crawlerConfig.Nip,
		Email:   crawlerConfig.NotificationEmail,
		Name:    fmt.Sprintf("%s %s", user.FirstName, user.LastName),
	}
}

func (s *Scheduler) crawlerLoop(crawler *Crawler) {
	for {
		select {
		case user := <-s.queueCh:
			// Get results
			results := crawler.Run(user)
			// Check if results changed
			newRes := getNewResults(user, results)
			if len(newRes) > 0 {
				log.Printf("New results for user %s", user.Email)
				if len(user.Email) > 0 {
					err := s.sendEmail(user, newRes)
					if err != nil {
						log.Println(err)
					}
				}
				// Update results
				for _, res := range results {
					// Ignore results with errors
					if res.Err == nil {
						user.Classes[res.ClassIndex].Results = res.Class.Results
						user.Classes[res.ClassIndex].Total = res.Class.Total
						user.Classes[res.ClassIndex].Final = res.Class.Final
					}
				}
			}

			err := s.UserResultsStore.UpdateResults(&api.Results{
				UserID:     user.ID,
				Classes:    user.Classes,
				LastUpdate: time.Now(),
			})
			if err != nil {
				log.Println(err)
			}
		}
	}
}

// The scheduler main loop
func (s *Scheduler) mainLoop() {
	ticker := time.NewTicker(checkInterval)
	for {
		select {
		case <-ticker.C:
			users, err := s.UserStore.ListUsers()
			if err != nil {
				log.Println(err)
				break
			}

			// Check which users need to update
			for _, user := range users {
				s.Queue(user)
			}
		case <-s.doneCh:
			// Stop the program
			return
		}
	}
}

func (s *Scheduler) sendEmail(user *crawlerUser, newResults []api.Class) error {
	var msg bytes.Buffer
	data := struct {
		User       *crawlerUser
		NewClasses []api.Class
	}{
		user,
		newResults,
	}
	err := msgTemplate.Execute(&msg, data)
	if err != nil {
		return err
	}

	return s.Sender.Send(user.Email, "You have new results!", string(msg.Bytes()))
}

func getNewResults(user *crawlerUser, newResults []runResult) []api.Class {
	var resClasses []api.Class
	for i, resInfo := range newResults {
		if resInfo.Err != nil {
			continue
		}

		var classChanged bool
		var curResults []api.Result
		for j, res := range resInfo.Class.Results {
			if user.Classes[i].Final != resInfo.Class.Final ||
				user.Classes[i].Total.Result != resInfo.Class.Total.Result ||
				user.Classes[i].Total.Average != resInfo.Class.Total.Average {
				classChanged = true
			}
			if len(user.Classes[i].Results) <= j {
				// If the is a new result
				curResults = append(curResults, res)
			} else {
				// Check if a result changed
				oldRes := user.Classes[i].Results[j]
				if oldRes.Name != res.Name ||
					oldRes.Normal.Average != res.Normal.Average ||
					oldRes.Normal.Result != res.Normal.Result {
					curResults = append(curResults, res)
				}
			}
		}
		if len(curResults) > 0 || classChanged {
			classInfo := user.Classes[i]
			resClasses = append(resClasses, api.Class{
				Name:    classInfo.Name,
				Group:   classInfo.Group,
				Year:    classInfo.Year,
				Results: curResults,
				Final:   resInfo.Class.Final,
				Total:   resInfo.Class.Total,
			})
		}
	}
	return resClasses
}
<commit_msg>fix time between updates<commit_after>package crawler

import (
	"bytes"
	"fmt"
	"log"
	"text/template"
	"time"

	"github.com/janicduplessis/resultscrawler/pkg/api"
	"github.com/janicduplessis/resultscrawler/pkg/store/crawlerconfig"
	"github.com/janicduplessis/resultscrawler/pkg/store/results"
	"github.com/janicduplessis/resultscrawler/pkg/store/user"
	"github.com/janicduplessis/resultscrawler/pkg/tools"
)

const (
	// Time between checks to see if a user needs an update in seconds
	checkInterval time.Duration = 30 * time.Second
	// Time between updates for each user in minutes
	updateInterval time.Duration = 10 * time.Minute
)

var msgTemplatePath = "msgtemplate.html"
var msgTemplate *template.Template

type crawlerUser struct {
	ID      string
	Classes []api.Class
	Nip     string
	Code    string
	Name    string
	Email   string
}

// SchedulerConfig initializes the scheduler.
type SchedulerConfig struct {
	Crawlers           []*Crawler
	UserStore          user.Store
	CrawlerConfigStore crawlerconfig.Store
	UserResultsStore   results.Store
	Sender             tools.Sender
}

// Scheduler handles scheduling crawler runs for every user.
type Scheduler struct {
	Crawlers           []*Crawler
	UserStore          user.Store
	CrawlerConfigStore crawlerconfig.Store
	UserResultsStore   results.Store
	Sender             tools.Sender

	queueCh chan *crawlerUser
	doneCh  chan bool
}

// NewScheduler creates a new scuduler object.
func NewScheduler(config *SchedulerConfig) *Scheduler {
	if msgTemplate == nil {
		msgTemplate = template.Must(template.New("msgtemplate.html").ParseFiles(msgTemplatePath))
	}

	queueCh := make(chan *crawlerUser)
	doneCh := make(chan bool)

	return &Scheduler{
		config.Crawlers,
		config.UserStore,
		config.CrawlerConfigStore,
		config.UserResultsStore,
		config.Sender,

		queueCh,
		doneCh,
	}
}

// Start starts the scheduler
func (s *Scheduler) Start() {
	for _, crawler := range s.Crawlers {
		go s.crawlerLoop(crawler)
	}

	s.mainLoop()
}

// Queue tells the scheduler do a run for a user
func (s *Scheduler) Queue(user *api.User) {
	// Get the user current results
	results, err := s.UserResultsStore.GetResults(user.ID)
	if err != nil {
		log.Println(err)
		return
	}

	// Check la update time.
	if time.Now().Sub(results.LastUpdate) < updateInterval {
		return
	}

	// Get crawler config.
	crawlerConfig, err := s.CrawlerConfigStore.GetCrawlerConfig(user.ID)
	if err != nil {
		log.Println(err)
		return
	}

	s.queueCh <- &crawlerUser{
		ID:      user.ID,
		Classes: results.Classes,
		Code:    crawlerConfig.Code,
		Nip:     crawlerConfig.Nip,
		Email:   crawlerConfig.NotificationEmail,
		Name:    fmt.Sprintf("%s %s", user.FirstName, user.LastName),
	}
}

func (s *Scheduler) crawlerLoop(crawler *Crawler) {
	for {
		select {
		case user := <-s.queueCh:
			// Get results
			results := crawler.Run(user)
			// Check if results changed
			newRes := getNewResults(user, results)
			if len(newRes) > 0 {
				log.Printf("New results for user %s", user.Email)
				if len(user.Email) > 0 {
					err := s.sendEmail(user, newRes)
					if err != nil {
						log.Println(err)
					}
				}
				// Update results
				for _, res := range results {
					// Ignore results with errors
					if res.Err == nil {
						user.Classes[res.ClassIndex].Results = res.Class.Results
						user.Classes[res.ClassIndex].Total = res.Class.Total
						user.Classes[res.ClassIndex].Final = res.Class.Final
					}
				}
			}

			err := s.UserResultsStore.UpdateResults(&api.Results{
				UserID:     user.ID,
				Classes:    user.Classes,
				LastUpdate: time.Now(),
			})
			if err != nil {
				log.Println(err)
			}
		}
	}
}

// The scheduler main loop
func (s *Scheduler) mainLoop() {
	ticker := time.NewTicker(checkInterval)
	for {
		select {
		case <-ticker.C:
			users, err := s.UserStore.ListUsers()
			if err != nil {
				log.Println(err)
				break
			}

			// Check which users need to update
			for _, user := range users {
				s.Queue(user)
			}
		case <-s.doneCh:
			// Stop the program
			return
		}
	}
}

func (s *Scheduler) sendEmail(user *crawlerUser, newResults []api.Class) error {
	var msg bytes.Buffer
	data := struct {
		User       *crawlerUser
		NewClasses []api.Class
	}{
		user,
		newResults,
	}
	err := msgTemplate.Execute(&msg, data)
	if err != nil {
		return err
	}

	return s.Sender.Send(user.Email, "You have new results!", string(msg.Bytes()))
}

func getNewResults(user *crawlerUser, newResults []runResult) []api.Class {
	var resClasses []api.Class
	for i, resInfo := range newResults {
		if resInfo.Err != nil {
			continue
		}

		var classChanged bool
		var curResults []api.Result
		for j, res := range resInfo.Class.Results {
			if user.Classes[i].Final != resInfo.Class.Final ||
				user.Classes[i].Total.Result != resInfo.Class.Total.Result ||
				user.Classes[i].Total.Average != resInfo.Class.Total.Average {
				classChanged = true
			}
			if len(user.Classes[i].Results) <= j {
				// If the is a new result
				curResults = append(curResults, res)
			} else {
				// Check if a result changed
				oldRes := user.Classes[i].Results[j]
				if oldRes.Name != res.Name ||
					oldRes.Normal.Average != res.Normal.Average ||
					oldRes.Normal.Result != res.Normal.Result {
					curResults = append(curResults, res)
				}
			}
		}
		if len(curResults) > 0 || classChanged {
			classInfo := user.Classes[i]
			resClasses = append(resClasses, api.Class{
				Name:    classInfo.Name,
				Group:   classInfo.Group,
				Year:    classInfo.Year,
				Results: curResults,
				Final:   resInfo.Class.Final,
				Total:   resInfo.Class.Total,
			})
		}
	}
	return resClasses
}
<|endoftext|>