<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"fmt"
	"time"

	federation_release_1_3 "k8s.io/kubernetes/federation/client/clientset_generated/federation_release_1_3"
	"k8s.io/kubernetes/pkg/api/errors"
	v1 "k8s.io/kubernetes/pkg/api/v1"
	cache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/controller"

	"github.com/golang/glog"
	"reflect"
	"sort"
)

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (sc *ServiceController) clusterServiceWorker() {
	fedClient := sc.federationClient
	for clusterName, cache := range sc.clusterCache.clientMap {
		go func(cache *clusterCache, clusterName string) {
			for {
				func() {
					key, quit := cache.serviceQueue.Get()
					defer cache.serviceQueue.Done(key)
					if quit {
						return
					}
					err := sc.clusterCache.syncService(key.(string), clusterName, cache, sc.serviceCache, fedClient, sc)
					if err != nil {
						glog.Errorf("Failed to sync service: %+v", err)
					}
				}()
			}
		}(cache, clusterName)
	}
}

// Whenever there is change on service, the federation service should be updated
func (cc *clusterClientCache) syncService(key, clusterName string, clusterCache *clusterCache, serviceCache *serviceCache, fedClient federation_release_1_3.Interface, sc *ServiceController) error {
	// obj holds the latest service info from apiserver, return if there is no federation cache for the service
	cachedService, ok := serviceCache.get(key)
	if !ok {
		// if serviceCache does not exists, that means the service is not created by federation, we should skip it
		return nil
	}
	serviceInterface, exists, err := clusterCache.serviceStore.GetByKey(key)
	if err != nil {
		glog.Infof("Did not successfully get %v from store: %v, will retry later", key, err)
		clusterCache.serviceQueue.Add(key)
		return err
	}
	var needUpdate bool
	if exists {
		service, ok := serviceInterface.(*v1.Service)
		if ok {
			glog.V(4).Infof("Found service for federation service %s/%s from cluster %s", service.Namespace, service.Name, clusterName)
			needUpdate = cc.processServiceUpdate(cachedService, service, clusterName)
		} else {
			_, ok := serviceInterface.(cache.DeletedFinalStateUnknown)
			if !ok {
				return fmt.Errorf("Object contained wasn't a service or a deleted key: %+v", serviceInterface)
			}
			glog.Infof("Found tombstone for %v", key)
			needUpdate = cc.processServiceDeletion(cachedService, clusterName)
		}
	} else {
		glog.Infof("Can not get service %v for cluster %s from serviceStore", key, clusterName)
		needUpdate = cc.processServiceDeletion(cachedService, clusterName)
	}

	if needUpdate {
		for i := 0; i < clientRetryCount; i++ {
			if err := sc.ensureDnsRecords(clusterName, cachedService); err == nil {
				break
			}
			glog.V(4).Infof("Error ensuring DNS Records for service %s on cluster %s: %v", key, clusterName, err)
			time.Sleep(cachedService.nextDNSUpdateDelay())
			clusterCache.serviceQueue.Add(key)
			// did not retry here as we still want to persist federation apiserver even ensure dns records fails
		}
		err := cc.persistFedServiceUpdate(cachedService, fedClient)
		if err == nil {
			cachedService.appliedState = cachedService.lastState
			cachedService.resetFedUpdateDelay()
		} else {
			if err != nil {
				glog.Errorf("Failed to sync service: %+v, put back to service queue", err)
				clusterCache.serviceQueue.Add(key)
			}
		}
	}
	return nil
}

// processServiceDeletion is triggered when a service is delete from underlying k8s cluster
// the deletion function will wip out the cached ingress info of the service from federation service ingress
// the function returns a bool to indicate if actual update happend on federation service cache
// and if the federation service cache is updated, the updated info should be post to federation apiserver
func (cc *clusterClientCache) processServiceDeletion(cachedService *cachedService, clusterName string) bool {
	cachedService.rwlock.Lock()
	defer cachedService.rwlock.Unlock()
	cachedStatus, ok := cachedService.serviceStatusMap[clusterName]
	// cached status found, remove ingress info from federation service cache
	if ok {
		cachedFedServiceStatus := cachedService.lastState.Status.LoadBalancer
		removeIndexes := []int{}
		for i, fed := range cachedFedServiceStatus.Ingress {
			for _, new := range cachedStatus.Ingress {
				// remove if same ingress record found
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					removeIndexes = append(removeIndexes, i)
				}
			}
		}
		sort.Ints(removeIndexes)
		for i := len(removeIndexes) - 1; i >= 0; i-- {
			cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress[:removeIndexes[i]], cachedFedServiceStatus.Ingress[removeIndexes[i]+1:]...)
			glog.V(4).Infof("Remove old ingress %d for service %s/%s", removeIndexes[i], cachedService.lastState.Namespace, cachedService.lastState.Name)
		}
		delete(cachedService.serviceStatusMap, clusterName)
		delete(cachedService.endpointMap, clusterName)
		cachedService.lastState.Status.LoadBalancer = cachedFedServiceStatus
		return true
	} else {
		glog.V(4).Infof("Service removal %s/%s from cluster %s observed.", cachedService.lastState.Namespace, cachedService.lastState.Name, clusterName)
	}
	return false
}

// processServiceUpdate Update ingress info when service updated
// the function returns a bool to indicate if actual update happend on federation service cache
// and if the federation service cache is updated, the updated info should be post to federation apiserver
func (cc *clusterClientCache) processServiceUpdate(cachedService *cachedService, service *v1.Service, clusterName string) bool {
	glog.V(4).Infof("Processing service update for %s/%s, cluster %s", service.Namespace, service.Name, clusterName)
	cachedService.rwlock.Lock()
	defer cachedService.rwlock.Unlock()
	var needUpdate bool
	newServiceLB := service.Status.LoadBalancer
	cachedFedServiceStatus := cachedService.lastState.Status.LoadBalancer
	if len(newServiceLB.Ingress) == 0 {
		// not yet get LB IP
		return false
	}

	cachedStatus, ok := cachedService.serviceStatusMap[clusterName]
	if ok {
		if reflect.DeepEqual(cachedStatus, newServiceLB) {
			glog.V(4).Infof("Same ingress info observed for service %s/%s: %+v ", service.Namespace, service.Name, cachedStatus.Ingress)
		} else {
			glog.V(4).Infof("Ingress info was changed for service %s/%s: cache: %+v, new: %+v ",
				service.Namespace, service.Name, cachedStatus.Ingress, newServiceLB)
			needUpdate = true
		}
	} else {
		glog.V(4).Infof("Cached service status was not found for %s/%s, cluster %s, building one", service.Namespace, service.Name, clusterName)

		// cache is not always reliable(cache will be cleaned when service controller restart)
		// two cases will run into this branch:
		// 1. new service loadbalancer info received -> no info in cache, and no in federation service
		// 2. service controller being restarted -> no info in cache, but it is in federation service

		// check if the lb info is already in federation service

		cachedService.serviceStatusMap[clusterName] = newServiceLB
		needUpdate = false
		// iterate service ingress info
		for _, new := range newServiceLB.Ingress {
			var found bool
			// if it is known by federation service
			for _, fed := range cachedFedServiceStatus.Ingress {
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					found = true
				}
			}
			if !found {
				needUpdate = true
				break
			}
		}
	}

	if needUpdate {
		// new status = cached federation status - cached status + new status from k8s cluster

		removeIndexes := []int{}
		for i, fed := range cachedFedServiceStatus.Ingress {
			for _, new := range cachedStatus.Ingress {
				// remove if same ingress record found
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					removeIndexes = append(removeIndexes, i)
				}
			}
		}
		sort.Ints(removeIndexes)
		for i := len(removeIndexes) - 1; i >= 0; i-- {
			cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress[:removeIndexes[i]], cachedFedServiceStatus.Ingress[removeIndexes[i]+1:]...)
		}
		cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress, service.Status.LoadBalancer.Ingress...)
		cachedService.lastState.Status.LoadBalancer = cachedFedServiceStatus
		glog.V(4).Infof("Add new ingress info %+v for service %s/%s", service.Status.LoadBalancer, service.Namespace, service.Name)
	} else {
		glog.V(4).Infof("Same ingress info found for %s/%s, cluster %s", service.Namespace, service.Name, clusterName)
	}
	return needUpdate
}

func (cc *clusterClientCache) persistFedServiceUpdate(cachedService *cachedService, fedClient federation_release_1_3.Interface) error {
	service := cachedService.lastState
	glog.V(5).Infof("Persist federation service status %s/%s", service.Namespace, service.Name)
	var err error
	for i := 0; i < clientRetryCount; i++ {
		_, err := fedClient.Core().Services(service.Namespace).Get(service.Name)
		if errors.IsNotFound(err) {
			glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v",
				service.Namespace, service.Name, err)
			return nil
		}
		_, err = fedClient.Core().Services(service.Namespace).UpdateStatus(service)
		if err == nil {
			glog.V(2).Infof("Successfully update service %s/%s to federation apiserver", service.Namespace, service.Name)
			return nil
		}
		if errors.IsNotFound(err) {
			glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v",
				service.Namespace, service.Name, err)
			return nil
		}
		if errors.IsConflict(err) {
			glog.V(4).Infof("Not persisting update to service '%s/%s' that has been changed since we received it: %v",
				service.Namespace, service.Name, err)
			return err
		}
		time.Sleep(cachedService.nextFedUpdateDelay())
	}
	return err
}

// obj could be an *api.Service, or a DeletionFinalStateUnknown marker item.
func (cc *clusterClientCache) enqueueService(obj interface{}, clusterName string) {
	key, err := controller.KeyFunc(obj)
	if err != nil {
		glog.Errorf("Couldn't get key for object %+v: %v", obj, err)
		return
	}
	_, ok := cc.clientMap[clusterName]
	if ok {
		cc.clientMap[clusterName].serviceQueue.Add(key)
	}
}
<commit_msg>Fix several spelling errors in comments.<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"fmt"
	"time"

	federation_release_1_3 "k8s.io/kubernetes/federation/client/clientset_generated/federation_release_1_3"
	"k8s.io/kubernetes/pkg/api/errors"
	v1 "k8s.io/kubernetes/pkg/api/v1"
	cache "k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/controller"

	"github.com/golang/glog"
	"reflect"
	"sort"
)

// worker runs a worker thread that just dequeues items, processes them, and marks them done.
// It enforces that the syncHandler is never invoked concurrently with the same key.
func (sc *ServiceController) clusterServiceWorker() {
	fedClient := sc.federationClient
	for clusterName, cache := range sc.clusterCache.clientMap {
		go func(cache *clusterCache, clusterName string) {
			for {
				func() {
					key, quit := cache.serviceQueue.Get()
					defer cache.serviceQueue.Done(key)
					if quit {
						return
					}
					err := sc.clusterCache.syncService(key.(string), clusterName, cache, sc.serviceCache, fedClient, sc)
					if err != nil {
						glog.Errorf("Failed to sync service: %+v", err)
					}
				}()
			}
		}(cache, clusterName)
	}
}

// Whenever there is change on service, the federation service should be updated
func (cc *clusterClientCache) syncService(key, clusterName string, clusterCache *clusterCache, serviceCache *serviceCache, fedClient federation_release_1_3.Interface, sc *ServiceController) error {
	// obj holds the latest service info from apiserver, return if there is no federation cache for the service
	cachedService, ok := serviceCache.get(key)
	if !ok {
		// if serviceCache does not exists, that means the service is not created by federation, we should skip it
		return nil
	}
	serviceInterface, exists, err := clusterCache.serviceStore.GetByKey(key)
	if err != nil {
		glog.Infof("Did not successfully get %v from store: %v, will retry later", key, err)
		clusterCache.serviceQueue.Add(key)
		return err
	}
	var needUpdate bool
	if exists {
		service, ok := serviceInterface.(*v1.Service)
		if ok {
			glog.V(4).Infof("Found service for federation service %s/%s from cluster %s", service.Namespace, service.Name, clusterName)
			needUpdate = cc.processServiceUpdate(cachedService, service, clusterName)
		} else {
			_, ok := serviceInterface.(cache.DeletedFinalStateUnknown)
			if !ok {
				return fmt.Errorf("Object contained wasn't a service or a deleted key: %+v", serviceInterface)
			}
			glog.Infof("Found tombstone for %v", key)
			needUpdate = cc.processServiceDeletion(cachedService, clusterName)
		}
	} else {
		glog.Infof("Can not get service %v for cluster %s from serviceStore", key, clusterName)
		needUpdate = cc.processServiceDeletion(cachedService, clusterName)
	}

	if needUpdate {
		for i := 0; i < clientRetryCount; i++ {
			if err := sc.ensureDnsRecords(clusterName, cachedService); err == nil {
				break
			}
			glog.V(4).Infof("Error ensuring DNS Records for service %s on cluster %s: %v", key, clusterName, err)
			time.Sleep(cachedService.nextDNSUpdateDelay())
			clusterCache.serviceQueue.Add(key)
			// did not retry here as we still want to persist federation apiserver even ensure dns records fails
		}
		err := cc.persistFedServiceUpdate(cachedService, fedClient)
		if err == nil {
			cachedService.appliedState = cachedService.lastState
			cachedService.resetFedUpdateDelay()
		} else {
			if err != nil {
				glog.Errorf("Failed to sync service: %+v, put back to service queue", err)
				clusterCache.serviceQueue.Add(key)
			}
		}
	}
	return nil
}

// processServiceDeletion is triggered when a service is delete from underlying k8s cluster
// the deletion function will wip out the cached ingress info of the service from federation service ingress
// the function returns a bool to indicate if actual update happened on federation service cache
// and if the federation service cache is updated, the updated info should be post to federation apiserver
func (cc *clusterClientCache) processServiceDeletion(cachedService *cachedService, clusterName string) bool {
	cachedService.rwlock.Lock()
	defer cachedService.rwlock.Unlock()
	cachedStatus, ok := cachedService.serviceStatusMap[clusterName]
	// cached status found, remove ingress info from federation service cache
	if ok {
		cachedFedServiceStatus := cachedService.lastState.Status.LoadBalancer
		removeIndexes := []int{}
		for i, fed := range cachedFedServiceStatus.Ingress {
			for _, new := range cachedStatus.Ingress {
				// remove if same ingress record found
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					removeIndexes = append(removeIndexes, i)
				}
			}
		}
		sort.Ints(removeIndexes)
		for i := len(removeIndexes) - 1; i >= 0; i-- {
			cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress[:removeIndexes[i]], cachedFedServiceStatus.Ingress[removeIndexes[i]+1:]...)
			glog.V(4).Infof("Remove old ingress %d for service %s/%s", removeIndexes[i], cachedService.lastState.Namespace, cachedService.lastState.Name)
		}
		delete(cachedService.serviceStatusMap, clusterName)
		delete(cachedService.endpointMap, clusterName)
		cachedService.lastState.Status.LoadBalancer = cachedFedServiceStatus
		return true
	} else {
		glog.V(4).Infof("Service removal %s/%s from cluster %s observed.", cachedService.lastState.Namespace, cachedService.lastState.Name, clusterName)
	}
	return false
}

// processServiceUpdate Update ingress info when service updated
// the function returns a bool to indicate if actual update happened on federation service cache
// and if the federation service cache is updated, the updated info should be post to federation apiserver
func (cc *clusterClientCache) processServiceUpdate(cachedService *cachedService, service *v1.Service, clusterName string) bool {
	glog.V(4).Infof("Processing service update for %s/%s, cluster %s", service.Namespace, service.Name, clusterName)
	cachedService.rwlock.Lock()
	defer cachedService.rwlock.Unlock()
	var needUpdate bool
	newServiceLB := service.Status.LoadBalancer
	cachedFedServiceStatus := cachedService.lastState.Status.LoadBalancer
	if len(newServiceLB.Ingress) == 0 {
		// not yet get LB IP
		return false
	}

	cachedStatus, ok := cachedService.serviceStatusMap[clusterName]
	if ok {
		if reflect.DeepEqual(cachedStatus, newServiceLB) {
			glog.V(4).Infof("Same ingress info observed for service %s/%s: %+v ", service.Namespace, service.Name, cachedStatus.Ingress)
		} else {
			glog.V(4).Infof("Ingress info was changed for service %s/%s: cache: %+v, new: %+v ",
				service.Namespace, service.Name, cachedStatus.Ingress, newServiceLB)
			needUpdate = true
		}
	} else {
		glog.V(4).Infof("Cached service status was not found for %s/%s, cluster %s, building one", service.Namespace, service.Name, clusterName)

		// cache is not always reliable(cache will be cleaned when service controller restart)
		// two cases will run into this branch:
		// 1. new service loadbalancer info received -> no info in cache, and no in federation service
		// 2. service controller being restarted -> no info in cache, but it is in federation service

		// check if the lb info is already in federation service

		cachedService.serviceStatusMap[clusterName] = newServiceLB
		needUpdate = false
		// iterate service ingress info
		for _, new := range newServiceLB.Ingress {
			var found bool
			// if it is known by federation service
			for _, fed := range cachedFedServiceStatus.Ingress {
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					found = true
				}
			}
			if !found {
				needUpdate = true
				break
			}
		}
	}

	if needUpdate {
		// new status = cached federation status - cached status + new status from k8s cluster

		removeIndexes := []int{}
		for i, fed := range cachedFedServiceStatus.Ingress {
			for _, new := range cachedStatus.Ingress {
				// remove if same ingress record found
				if new.IP == fed.IP && new.Hostname == fed.Hostname {
					removeIndexes = append(removeIndexes, i)
				}
			}
		}
		sort.Ints(removeIndexes)
		for i := len(removeIndexes) - 1; i >= 0; i-- {
			cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress[:removeIndexes[i]], cachedFedServiceStatus.Ingress[removeIndexes[i]+1:]...)
		}
		cachedFedServiceStatus.Ingress = append(cachedFedServiceStatus.Ingress, service.Status.LoadBalancer.Ingress...)
		cachedService.lastState.Status.LoadBalancer = cachedFedServiceStatus
		glog.V(4).Infof("Add new ingress info %+v for service %s/%s", service.Status.LoadBalancer, service.Namespace, service.Name)
	} else {
		glog.V(4).Infof("Same ingress info found for %s/%s, cluster %s", service.Namespace, service.Name, clusterName)
	}
	return needUpdate
}

func (cc *clusterClientCache) persistFedServiceUpdate(cachedService *cachedService, fedClient federation_release_1_3.Interface) error {
	service := cachedService.lastState
	glog.V(5).Infof("Persist federation service status %s/%s", service.Namespace, service.Name)
	var err error
	for i := 0; i < clientRetryCount; i++ {
		_, err := fedClient.Core().Services(service.Namespace).Get(service.Name)
		if errors.IsNotFound(err) {
			glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v",
				service.Namespace, service.Name, err)
			return nil
		}
		_, err = fedClient.Core().Services(service.Namespace).UpdateStatus(service)
		if err == nil {
			glog.V(2).Infof("Successfully update service %s/%s to federation apiserver", service.Namespace, service.Name)
			return nil
		}
		if errors.IsNotFound(err) {
			glog.Infof("Not persisting update to service '%s/%s' that no longer exists: %v",
				service.Namespace, service.Name, err)
			return nil
		}
		if errors.IsConflict(err) {
			glog.V(4).Infof("Not persisting update to service '%s/%s' that has been changed since we received it: %v",
				service.Namespace, service.Name, err)
			return err
		}
		time.Sleep(cachedService.nextFedUpdateDelay())
	}
	return err
}

// obj could be an *api.Service, or a DeletionFinalStateUnknown marker item.
func (cc *clusterClientCache) enqueueService(obj interface{}, clusterName string) {
	key, err := controller.KeyFunc(obj)
	if err != nil {
		glog.Errorf("Couldn't get key for object %+v: %v", obj, err)
		return
	}
	_, ok := cc.clientMap[clusterName]
	if ok {
		cc.clientMap[clusterName].serviceQueue.Add(key)
	}
}
<|endoftext|>


<commit_before>// Lab2_file_database project main.go
package main

import (
	"bufio"
	"bytes"
	"encoding/csv"
	"encoding/xml"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

type cache struct {
	tables []table
}

type Person struct {
	XMLName xml.Name `xml:"person"`
	Name    string   `xml:"name"`
	Age     int      `xml:"age"`
}

type People struct {
	XMLName xml.Name `xml:"people"`
	People  []Person `xml:"person"`
}

func ReadEntries(reader io.Reader) ([]Person, error) {
	var people People
	if err := xml.NewDecoder(reader).Decode(&people); err != nil {
		return nil, err
	}

	return people.People, nil
}

func FlushDatabase(writer io.Writer, xmlEntries People) error {
	if err := xml.NewEncoder(writer).Encode(xmlEntries); err != nil {
		return err
	}

	return nil
}

func (c *cache) get(name string) *table {
	//Проверяем есть ли таблицца на оперативке
	fmt.Println("Try to get table")
	for _, t := range c.tables {
		if t.name == name {
			fmt.Println("table find in cache")
			return &t
		}
	}
	//Пытаемся загрузить базу из файлововой системы
	f, err := os.Open(name)
	if err == nil {
		data := make([][]string, 0, 0)
		r := csv.NewReader(bufio.NewReader(f))

		for {
			record, err := r.Read()
			// Stop at EOF.
			if err == io.EOF {
				break
			}

			row_data := []string{record[0], record[1]}
			data = append(data, row_data)
		}
		t := table{name: name, data: &data}
		c.tables = append(c.tables, t)
		return &t
	}
	//Ну и если ни черта не вышло создаем пустую базу
	t := table{name: name, data: new([][]string)}
	c.tables = append(c.tables, t)
	fmt.Println("table returned")
	return &t
}

type table struct {
	isLocked sync.Mutex
	name     string
	data     *[][]string
}

func (t *table) save() {
	//	for t.isLocked {

	//	}

	t.isLocked.Lock()
	file, _ := os.Create(t.name)
	defer file.Close()
	for _, row := range *t.data {
		file.WriteString(row[0] + "," + row[1] + "\n")
	}
	t.isLocked.Unlock()
}

const (
	CONN_HOST = "localhost"
	CONN_PORT = "8888"
	CONN_TYPE = "tcp"
)

//func main() {
//	// Listen for incoming connections.
//	c := cache{}
//	l, err := net.Listen(CONN_TYPE, CONN_HOST+":"+CONN_PORT)
//	if err != nil {
//		fmt.Println("Error listening:", err.Error())
//		os.Exit(1)
//	}
//	// Close the listener when the application closes.
//	defer l.Close()
//	fmt.Println("Listening on " + CONN_HOST + ":" + CONN_PORT)
//	for {
//		// Listen for an incoming connection.
//		conn, err := l.Accept()
//		fmt.Println("accept")
//
//		if err != nil {
//			fmt.Println("Error accepting: ", err.Error())
//			os.Exit(1)
//		}
//		// Handle connections in a new goroutine.
//		go handleConnection(conn, &c)
//	}
//}

// Handles incoming requests.
func handleConnection(conn net.Conn, c *cache) {
	// Make a buffer to hold incoming data.
	var buf [512]byte
	empty_string := ""
	for {
		copy(buf[:], empty_string) //make  buffer empty
		_, err := conn.Read(buf[0:])
		if err != nil {
			return
		}
		n := bytes.Index(buf[:], []byte{0})
		s := string(buf[:n])
		go handleRequest(s, c, conn)
	}
}

func handleRequest(command string, c *cache, conn net.Conn) {
	fmt.Println("handle request: " + command)

	params := strings.Fields(command)
	// remove empty entries and remove whitespaces
	fmt.Println(params)
	fmt.Println(params[0])
	// remove empty entries and remove whitespaces
	switch strings.ToLower(params[0]) {
	case "exit":
		os.Exit(0)
	case "insert":
		fmt.Println("switching")

		fmt.Println("insert")
		insert_regex := regexp.MustCompile(`^insert\s+\w+\s+\w+\s+into\s+[A-z]+[A-z_0-9]*`)
		if insert_regex.MatchString(command) {
			t := c.get(params[4])
			t.insert_(params[1], params[2])

		} else {
			conn.Write([]byte(string("Your command didn't match the pattern\n")))
		}
	case "select":
		fmt.Println("try to select")
		select_regex := regexp.MustCompile(`^select\s+\w+\s+from\s+[A-z]+[A-z_0-9]*`)
		if select_regex.MatchString(command) {

			t := c.get(params[3])
			data := ""
			for _, elem := range t.select_(params[1]) {
				data += elem + " "
				conn.Write([]byte(string(elem) + " "))

			}
			conn.Write([]byte("\n"))

			fmt.Println(data + "\n newline was sended")
		} else {
			conn.Write([]byte(string("Your command didn't match the pattern\n")))
		}
		fmt.Println("select")
	case "delete":
		delete_regex := regexp.MustCompile(`^delete\s+\w+\s+in\s+[A-z]+[A-z_0-9]*`)
		if delete_regex.MatchString(command) {
			t := c.get(params[3])
			t.delete_(params[1])
		} else {
			conn.Write([]byte(string("Your command didn't match the pattern\n")))
		}
		fmt.Println("delete")

	case "update":
		update_regex := regexp.MustCompile(`^update\s+\w+\s+to\s+\w+\s+in\s+[A-z]+[A-z_0-9]*`)
		if update_regex.MatchString(command) {
			t := c.get(params[5])
			t.update_(params[1], params[3])
		} else {
			conn.Write([]byte(string("Your command didn't match the pattern\n")))
		}
		fmt.Println("update")
	default:
		conn.Write([]byte(string("Your command didn't match the pattern\n")))
	}
}

func (t *table) insert_(key string, value string) {
	row := []string{key, value}
	//Возможное место для мютекса
	fmt.Println(t.data)
	t.additem(row)
	fmt.Println(t.data)
	go t.save()
}
func (t *table) additem(value []string) {
	*t.data = append(*t.data, value)
}

func (t *table) select_(key string) []string {
	selecting_data := []string{}
	fmt.Println(*t.data)
	for _, row := range *t.data {
		if row[0] == key {
			selecting_data = append(selecting_data, row[1])
		}
	}

	return selecting_data
}

func (p *People) selectByKey(key string) []int {
	selecting_data := []int{}

	for _, item := range p.People {
		if item.Name == key {
			selecting_data = append(selecting_data, item.Age)
		}
	}
	return selecting_data
}

func (p *People) deleteByKey(key string) {
	for index := 0; index < len(p.People); index++ {
		if p.People[index].Name == key {
			p.People = append((p.People)[:index], (p.People)[index+1:]...)
		}
	}
}

func (p *People) updateByKey(key string, value int) {
	for index := 0; index < len(p.People); index++ {
		if p.People[index].Name == key {
			p.People[index].Age = value
		}
	}
}

func (p *People) addItem(key string, value int) {
	human := Person{Name: key, Age: value}
	p.People = append(p.People, human)
}

func (t *table) delete_(key string) {
	for index := 0; index < len(*t.data); index++ {
		if (*t.data)[index][0] == key {
			*t.data = append((*t.data)[:index], (*t.data)[index+1:]...)
		}
	}
	go t.save()
}

func (t *table) update_(key string, value string) {
	for i, row := range *t.data {
		if row[0] == key {
			(*t.data)[i][1] = value
		}

	}
	go t.save()
}

func main() {
	// Build the location of the straps.xml file
	// filepath.Abs appends the file name to the default working directly
	strapsFilePath, err := filepath.Abs("entries.xml")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Open the container file
	file, err := os.Open(strapsFilePath)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Parse entries
	entries, err := ReadEntries(file)
	file.Close()

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	a := Person{Name: "Vasya", Age: 25}

	entries = append(entries, a)

	humans := People{People: entries}

	fmt.Print("++++++++++++++++++++++++++++++++++++++++++++")
	tmp := humans.selectByKey("Vasya")
	fmt.Printf("Age: %2d\n", tmp)
	humans.updateByKey("Vasya", 34)
	fmt.Print("++++++++++++++++++++++++++++++++++++++++++++")
	tmp = humans.selectByKey("Vasya")
	fmt.Printf("Age: %2d\n", tmp)
	humans.deleteByKey("Vasya")
	fmt.Print("++++++++++++++++++++++++++++++++++++++++++++")
	tmp = humans.selectByKey("Vasya")
	fmt.Printf("Age: %2d\n", tmp)
	humans.addItem("Vasya", 25)
	fmt.Print("++++++++++++++++++++++++++++++++++++++++++++")
	tmp = humans.selectByKey("Vasya")
	fmt.Printf("Age: %2d\n", tmp)
	res := People{People: entries}
	fileToWrite, err := os.Create("result.xml")

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	FlushDatabase(fileToWrite, res)

	for _, person := range humans.People {
		// index is the index where we are
		// element is the element from someSlice for where we are
		fmt.Printf("Name: %6s  Age: %2d\n", person.Name, person.Age)
	}
	// Display The first strap
}
<commit_msg>refactoring<commit_after>// Lab2_file_database project main.go
package main

import (
	"bufio"
	"bytes"
	"encoding/csv"
	"encoding/xml"
	"fmt"
	"io"
	"net"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
)

type cache struct {
	tables []table
}

type Person struct {
	XMLName xml.Name `xml:"person"`
	Name    string   `xml:"name"`
	Age     int      `xml:"age"`
}

type People struct {
	XMLName xml.Name `xml:"people"`
	People  []Person `xml:"person"`
}

func ReadEntries(reader io.Reader) ([]Person, error) {
	var people People
	if err := xml.NewDecoder(reader).Decode(&people); err != nil {
		return nil, err
	}

	return people.People, nil
}

func FlushDatabase(writer io.Writer, xmlEntries People) error {
	if err := xml.NewEncoder(writer).Encode(xmlEntries); err != nil {
		return err
	}

	return nil
}

func (c *cache) get(name string) *table {
	//Проверяем есть ли таблицца на оперативке
	fmt.Println("Try to get table")
	for _, t := range c.tables {
		if t.name == name {
			fmt.Println("table find in cache")
			return &t
		}
	}
	//Пытаемся загрузить базу из файлововой системы
	f, err := os.Open(name)
	if err == nil {
		data := make([][]string, 0, 0)
		r := csv.NewReader(bufio.NewReader(f))

		for {
			record, err := r.Read()
			// Stop at EOF.
			if err == io.EOF {
				break
			}

			row_data := []string{record[0], record[1]}
			data = append(data, row_data)
		}
		t := table{name: name, data: &data}
		c.tables = append(c.tables, t)
		return &t
	}
	//Ну и если ни черта не вышло создаем пустую базу
	t := table{name: name, data: new([][]string)}
	c.tables = append(c.tables, t)
	fmt.Println("table returned")
	return &t
}

type table struct {
	isLocked sync.Mutex
	name     string
	data     *[][]string
}

func (t *table) save() {
	//	for t.isLocked {

	//	}

	t.isLocked.Lock()
	file, _ := os.Create(t.name)
	defer file.Close()
	for _, row := range *t.data {
		file.WriteString(row[0] + "," + row[1] + "\n")
	}
	t.isLocked.Unlock()
}

const (
	CONN_HOST = "localhost"
	CONN_PORT = "8888"
	CONN_TYPE = "tcp"
)

//func main() {
//	// Listen for incoming connections.
//	c := cache{}
//	l, err := net.Listen(CONN_TYPE, CONN_HOST+":"+CONN_PORT)
//	if err != nil {
//		fmt.Println("Error listening:", err.Error())
//		os.Exit(1)
//	}
//	// Close the listener when the application closes.
//	defer l.Close()
//	fmt.Println("Listening on " + CONN_HOST + ":" + CONN_PORT)
//	for {
//		// Listen for an incoming connection.
//		conn, err := l.Accept()
//		fmt.Println("accept")
//
//		if err != nil {
//			fmt.Println("Error accepting: ", err.Error())
//			os.Exit(1)
//		}
//		// Handle connections in a new goroutine.
//		go handleConnection(conn, &c)
//	}
//}

// Handles incoming requests.
func handleConnection(conn net.Conn, c *cache) {
	// Make a buffer to hold incoming data.
	var buf [512]byte
	empty_string := ""
	for {
		copy(buf[:], empty_string) //make  buffer empty
		_, err := conn.Read(buf[0:])
		if err != nil {
			return
		}
		n := bytes.Index(buf[:], []byte{0})
		s := string(buf[:n])
		go handleRequest(s, c, conn)
	}
}

func handleRequest(command string, c *cache, conn net.Conn) {
	fmt.Println("handle request: " + command)

	params := strings.Fields(command)
	// remove empty entries and remove whitespaces
	fmt.Println(params)
	fmt.Println(params[0])
	// remove empty entries and remove whitespaces
	switch strings.ToLower(params[0]) {
	case "exit":
		os.Exit(0)
	case "insert":
		fmt.Println("switching")

		fmt.Println("insert")
		insert_regex := regexp.MustCompile(`^insert\s+\w+\s+\w+\s+into\s+[A-z]+[A-z_0-9]*`)
		if insert_regex.MatchString(command) {
			t := c.get(params[4])
			t.insert_(params[1], params[2])

		} else {
			conn.Write([]byte(string("Your command didn't match the pattern\n")))
		}
	case "select":
		fmt.Println("try to select")
		select_regex := regexp.MustCompile(`^select\s+\w+\s+from\s+[A-z]+[A-z_0-9]*`)
		if select_regex.MatchString(command) {

			t := c.get(params[3])
			data := ""
			for _, elem := range t.select_(params[1]) {
				data += elem + " "
				conn.Write([]byte(string(elem) + " "))

			}
			conn.Write([]byte("\n"))

			fmt.Println(data + "\n newline was sended")
		} else {
			conn.Write([]byte(string("Your command didn't match the pattern\n")))
		}
		fmt.Println("select")
	case "delete":
		delete_regex := regexp.MustCompile(`^delete\s+\w+\s+in\s+[A-z]+[A-z_0-9]*`)
		if delete_regex.MatchString(command) {
			t := c.get(params[3])
			t.delete_(params[1])
		} else {
			conn.Write([]byte(string("Your command didn't match the pattern\n")))
		}
		fmt.Println("delete")

	case "update":
		update_regex := regexp.MustCompile(`^update\s+\w+\s+to\s+\w+\s+in\s+[A-z]+[A-z_0-9]*`)
		if update_regex.MatchString(command) {
			t := c.get(params[5])
			t.update_(params[1], params[3])
		} else {
			conn.Write([]byte(string("Your command didn't match the pattern\n")))
		}
		fmt.Println("update")
	default:
		conn.Write([]byte(string("Your command didn't match the pattern\n")))
	}
}

func (t *table) insert_(key string, value string) {
	row := []string{key, value}
	//Возможное место для мютекса
	fmt.Println(t.data)
	t.additem(row)
	fmt.Println(t.data)
	go t.save()
}
func (t *table) additem(value []string) {
	*t.data = append(*t.data, value)
}

func (t *table) select_(key string) []string {
	selecting_data := []string{}
	fmt.Println(*t.data)
	for _, row := range *t.data {
		if row[0] == key {
			selecting_data = append(selecting_data, row[1])
		}
	}

	return selecting_data
}

func (p *People) selectByKey(key name) []int {
	selecting_data := []int{}

	for _, item := range p.People {
		if item.Name == name {
			selecting_data = append(selecting_data, item.Age)
		}
	}
	return selecting_data
}

func (p *People) deleteByKey(name string) {
	for index := 0; index < len(p.People); index++ {
		if p.People[index].Name == name {
			p.People = append((p.People)[:index], (p.People)[index+1:]...)
		}
	}
}

func (p *People) updateByKey(name string, age int) {
	for index := 0; index < len(p.People); index++ {
		if p.People[index].Name == name {
			p.People[index].Age = age
		}
	}
}

func (p *People) addItem(name string, age int) {
	human := Person{Name: name, Age: age}
	p.People = append(p.People, human)
}

func (t *table) delete_(key string) {
	for index := 0; index < len(*t.data); index++ {
		if (*t.data)[index][0] == key {
			*t.data = append((*t.data)[:index], (*t.data)[index+1:]...)
		}
	}
	go t.save()
}

func (t *table) update_(key string, value string) {
	for i, row := range *t.data {
		if row[0] == key {
			(*t.data)[i][1] = value
		}

	}
	go t.save()
}

func main() {
	// Build the location of the straps.xml file
	// filepath.Abs appends the file name to the default working directly
	strapsFilePath, err := filepath.Abs("entries.xml")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Open the container file
	file, err := os.Open(strapsFilePath)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// Parse entries
	entries, err := ReadEntries(file)
	file.Close()

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	a := Person{Name: "Vasya", Age: 25}

	entries = append(entries, a)

	humans := People{People: entries}

	fmt.Print("++++++++++++++++++++++++++++++++++++++++++++")
	tmp := humans.selectByKey("Vasya")
	fmt.Printf("Age: %2d\n", tmp)
	humans.updateByKey("Vasya", 34)
	fmt.Print("++++++++++++++++++++++++++++++++++++++++++++")
	tmp = humans.selectByKey("Vasya")
	fmt.Printf("Age: %2d\n", tmp)
	humans.deleteByKey("Vasya")
	fmt.Print("++++++++++++++++++++++++++++++++++++++++++++")
	tmp = humans.selectByKey("Vasya")
	fmt.Printf("Age: %2d\n", tmp)
	humans.addItem("Vasya", 25)
	fmt.Print("++++++++++++++++++++++++++++++++++++++++++++")
	tmp = humans.selectByKey("Vasya")
	fmt.Printf("Age: %2d\n", tmp)
	res := People{People: entries}
	fileToWrite, err := os.Create("result.xml")

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	FlushDatabase(fileToWrite, res)

	for _, person := range humans.People {
		// index is the index where we are
		// element is the element from someSlice for where we are
		fmt.Printf("Name: %6s  Age: %2d\n", person.Name, person.Age)
	}
	// Display The first strap
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var g_odir = flag.String("o", "/afs/cern.ch/sw/lcg/contrib/go", "output directory where to install the go runtime")
var g_verbose = flag.Bool("v", false, "enable verbose printouts")
var g_mode = flag.String("mode", "curl", "which mode to use (go|curl)")

const (
	url_tmpl = "http://golang.org/dl/go%s.%s-%s.tar.gz"
)

func curl_download(version string, platform [2]string) error {
	odir := filepath.Join(*g_odir, version, "tmp-"+platform[0]+"-"+platform[1])

	// create out-dir layout
	if *g_verbose {
		fmt.Printf("~~~ %s\n", odir)
	}
	err := os.MkdirAll(odir, 0755)
	if err != nil {
		return err
	}

	// http://golang.org/dl/go1.2.2.linux-amd64.tar.gz
	url := fmt.Sprintf(
		url_tmpl,
		version,
		platform[0],
		platform[1],
	)

	curl := exec.Command(
		"curl",
		"-L", url,
	)

	untar := exec.Command(
		"tar",
		"-C", odir,
		"-zxf",
		"-",
	)

	if *g_verbose {
		fmt.Printf("curl.cmd:    %v\n", curl.Args)
		fmt.Printf("untar.cmd:   %v\n", untar.Args)
	}

	curl_out, err := curl.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** curl.cmd:    %v\n", curl.Args)
		fmt.Fprintf(os.Stderr, "**error** curl.stdout: %v\n", err)
		return err
	}

	err = curl.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** curl.cmd:   %v\n", curl.Args)
		fmt.Fprintf(os.Stderr, "**error** curl.start: %v\n", err)
		return err
	}

	untar.Stdin = curl_out

	err = untar.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** untar.cmd:   %v\n", untar.Args)
		fmt.Fprintf(os.Stderr, "**error** untar.start: %v\n", err)
		return err
	}

	err = curl.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** curl.cmd:   %v\n", curl.Args)
		fmt.Fprintf(os.Stderr, "**error** curl.start: %v\n", err)
		return err
	}

	err = untar.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** untar.cmd:  %v\n", untar.Args)
		fmt.Fprintf(os.Stderr, "**error** untar.wait: %v\n", err)
		return err
	}

	err = os.Rename(
		filepath.Join(odir, "go"),
		filepath.Join(*g_odir, version, platform[0]+"_"+platform[1]),
	)
	if err != nil {
		return err
	}
	err = os.RemoveAll(odir)
	if err != nil {
		return err
	}

	goroot := filepath.Join(*g_odir, version, platform[0]+"_"+platform[1])
	return create_setup_files(goroot, platform)
}

func go_download(version string, platform [2]string) error {
	odir := filepath.Join(*g_odir, version, "tmp-"+platform[0]+"-"+platform[1])

	// create out-dir layout
	if *g_verbose {
		fmt.Printf("~~~ %s\n", odir)
	}
	err := os.MkdirAll(odir, 0755)
	if err != nil {
		return err
	}

	// http://golang.org/dl/go1.1.2.linux-amd64.tar.gz
	url := fmt.Sprintf(
		url_tmpl,
		version,
		platform[0],
		platform[1],
	)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	gz, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	defer gz.Close()

	tr := tar.NewReader(gz)
	// Iterate through the files in the archive.
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			fmt.Fprintf(os.Stderr, "**error** %v\n", err)
			continue
		}

		switch hdr.Typeflag {
		case tar.TypeDir:
			dir := filepath.Join(odir, hdr.Name)
			if *g_verbose {
				fmt.Printf(">>> %s\n", dir)
			}
			err = os.MkdirAll(dir, 0755)
			if err != nil {
				return err
			}
			continue

		case tar.TypeReg, tar.TypeRegA:
			// ok
		default:
			fmt.Fprintf(os.Stderr, "**error: %v\n", hdr.Typeflag)
			return err
		}
		oname := filepath.Join(odir, hdr.Name)
		if *g_verbose {
			fmt.Printf("::: %s (%s)\n", oname, string(byte(hdr.Typeflag)))
		}
		dir := filepath.Dir(oname)
		err = os.MkdirAll(dir, 0755)
		if err != nil {
			return err
		}

		o, err := os.OpenFile(oname,
			os.O_WRONLY|os.O_CREATE,
			os.FileMode(hdr.Mode),
		)
		if err != nil {
			return err
		}
		defer o.Close()
		_, err = io.Copy(o, tr)
		if err != nil {
			return err
		}
		o.Sync()
		o.Close()
	}

	err = os.Rename(
		filepath.Join(odir, "go"),
		filepath.Join(*g_odir, version, platform[0]+"_"+platform[1]),
	)
	if err != nil {
		return err
	}
	err = os.RemoveAll(odir)
	if err != nil {
		return err
	}

	goroot := filepath.Join(*g_odir, version, platform[0]+"_"+platform[1])
	// dont hard-code references to the AFS rw-volume
	if strings.HasPrefix(goroot, "/afs/.") {
		goroot = strings.Replace(goroot, "/afs/.", "/afs/", 1)
	}
	return create_setup_files(goroot, platform)
}

func create_setup_files(goroot string, platform [2]string) error {
	// create setup.sh
	setup_sh, err := os.Create(
		filepath.Join(goroot, "setup.sh"),
	)
	if err != nil {
		return err
	}
	defer setup_sh.Close()
	_, err = setup_sh.WriteString(fmt.Sprintf(`#!/bin/sh

export GOROOT=%s
export PATH=${GOROOT}/bin:${PATH}
export GOPATH=${HOME}/dev/gocode
export PATH=${GOPATH}/bin:${PATH}
`, goroot))
	if err != nil {
		return err
	}

	// create setup.csh
	setup_csh, err := os.Create(
		filepath.Join(goroot, "setup.csh"),
	)
	if err != nil {
		return err
	}
	defer setup_csh.Close()
	_, err = setup_csh.WriteString(fmt.Sprintf(`
setenv GOROOT %s
setenv PATH ${GROOT}/bin:${PATH}
setenv GOPATH ${HOME}/dev/gocode
setenv PATH ${GOPATH}/bin:${PATH}
`))
	if err != nil {
		return err
	}

	return err
}

func main() {
	flag.Parse()
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `install-go installs the go-gc runtime.
Usage:

$ install-go [options] <go-version>
`)
		flag.PrintDefaults()
	}

	if flag.NArg() <= 0 {
		flag.Usage()
		os.Exit(1)
	}

	version := flag.Arg(0)
	if version == "" {
		flag.Usage()
		os.Exit(1)
	}

	platforms := [][2]string{
		{"linux", "amd64"},
		{"linux", "386"},
	}

	type download_fct func(version string, platform [2]string) error
	download := go_download

	switch *g_mode {
	case "curl":
		download = curl_download
	case "go":
		download = go_download
	default:
		fmt.Fprintf(os.Stderr, "**error** invalid download mode (%s)\n", *g_mode)
	}

	all_good := true
	errch := make(chan error, len(platforms))
	for _, plat := range platforms {
		go func(plat [2]string) {
			fmt.Printf(":: installing %s-%s...\n", plat[0], plat[1])
			err := download(version, plat)
			errch <- err
			if err != nil {
				return
			}
			fmt.Printf(":: installing %s-%s... [ok]\n", plat[0], plat[1])
		}(plat)
	}

	for i := 0; i < len(platforms); i++ {
		err := <-errch
		if err != nil {
			all_good = false
			fmt.Fprintf(os.Stderr, "**error** %v\n", err)
		}
		if i == len(platforms)-1 {
			close(errch)
		}
	}

	os.Remove(filepath.Join(*g_odir, "latest"))

	err := os.Symlink(
		version,
		filepath.Join(*g_odir, "latest"),
	)
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** %v\n", err)
		os.Exit(1)
	}

	if !all_good {
		os.Exit(1)
	}
}

// EOF
<commit_msg>main: fixup (t)csh setup script<commit_after>package main

import (
	"archive/tar"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

var g_odir = flag.String("o", "/afs/cern.ch/sw/lcg/contrib/go", "output directory where to install the go runtime")
var g_verbose = flag.Bool("v", false, "enable verbose printouts")
var g_mode = flag.String("mode", "curl", "which mode to use (go|curl)")

const (
	url_tmpl = "http://golang.org/dl/go%s.%s-%s.tar.gz"
)

func curl_download(version string, platform [2]string) error {
	odir := filepath.Join(*g_odir, version, "tmp-"+platform[0]+"-"+platform[1])

	// create out-dir layout
	if *g_verbose {
		fmt.Printf("~~~ %s\n", odir)
	}
	err := os.MkdirAll(odir, 0755)
	if err != nil {
		return err
	}

	// http://golang.org/dl/go1.2.2.linux-amd64.tar.gz
	url := fmt.Sprintf(
		url_tmpl,
		version,
		platform[0],
		platform[1],
	)

	curl := exec.Command(
		"curl",
		"-L", url,
	)

	untar := exec.Command(
		"tar",
		"-C", odir,
		"-zxf",
		"-",
	)

	if *g_verbose {
		fmt.Printf("curl.cmd:    %v\n", curl.Args)
		fmt.Printf("untar.cmd:   %v\n", untar.Args)
	}

	curl_out, err := curl.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** curl.cmd:    %v\n", curl.Args)
		fmt.Fprintf(os.Stderr, "**error** curl.stdout: %v\n", err)
		return err
	}

	err = curl.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** curl.cmd:   %v\n", curl.Args)
		fmt.Fprintf(os.Stderr, "**error** curl.start: %v\n", err)
		return err
	}

	untar.Stdin = curl_out

	err = untar.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** untar.cmd:   %v\n", untar.Args)
		fmt.Fprintf(os.Stderr, "**error** untar.start: %v\n", err)
		return err
	}

	err = curl.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** curl.cmd:   %v\n", curl.Args)
		fmt.Fprintf(os.Stderr, "**error** curl.start: %v\n", err)
		return err
	}

	err = untar.Wait()
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** untar.cmd:  %v\n", untar.Args)
		fmt.Fprintf(os.Stderr, "**error** untar.wait: %v\n", err)
		return err
	}

	err = os.Rename(
		filepath.Join(odir, "go"),
		filepath.Join(*g_odir, version, platform[0]+"_"+platform[1]),
	)
	if err != nil {
		return err
	}
	err = os.RemoveAll(odir)
	if err != nil {
		return err
	}

	goroot := filepath.Join(*g_odir, version, platform[0]+"_"+platform[1])
	return create_setup_files(goroot, platform)
}

func go_download(version string, platform [2]string) error {
	odir := filepath.Join(*g_odir, version, "tmp-"+platform[0]+"-"+platform[1])

	// create out-dir layout
	if *g_verbose {
		fmt.Printf("~~~ %s\n", odir)
	}
	err := os.MkdirAll(odir, 0755)
	if err != nil {
		return err
	}

	// http://golang.org/dl/go1.1.2.linux-amd64.tar.gz
	url := fmt.Sprintf(
		url_tmpl,
		version,
		platform[0],
		platform[1],
	)
	resp, err := http.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	gz, err := gzip.NewReader(resp.Body)
	if err != nil {
		return err
	}
	defer gz.Close()

	tr := tar.NewReader(gz)
	// Iterate through the files in the archive.
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			fmt.Fprintf(os.Stderr, "**error** %v\n", err)
			continue
		}

		switch hdr.Typeflag {
		case tar.TypeDir:
			dir := filepath.Join(odir, hdr.Name)
			if *g_verbose {
				fmt.Printf(">>> %s\n", dir)
			}
			err = os.MkdirAll(dir, 0755)
			if err != nil {
				return err
			}
			continue

		case tar.TypeReg, tar.TypeRegA:
			// ok
		default:
			fmt.Fprintf(os.Stderr, "**error: %v\n", hdr.Typeflag)
			return err
		}
		oname := filepath.Join(odir, hdr.Name)
		if *g_verbose {
			fmt.Printf("::: %s (%s)\n", oname, string(byte(hdr.Typeflag)))
		}
		dir := filepath.Dir(oname)
		err = os.MkdirAll(dir, 0755)
		if err != nil {
			return err
		}

		o, err := os.OpenFile(oname,
			os.O_WRONLY|os.O_CREATE,
			os.FileMode(hdr.Mode),
		)
		if err != nil {
			return err
		}
		defer o.Close()
		_, err = io.Copy(o, tr)
		if err != nil {
			return err
		}
		o.Sync()
		o.Close()
	}

	err = os.Rename(
		filepath.Join(odir, "go"),
		filepath.Join(*g_odir, version, platform[0]+"_"+platform[1]),
	)
	if err != nil {
		return err
	}
	err = os.RemoveAll(odir)
	if err != nil {
		return err
	}

	goroot := filepath.Join(*g_odir, version, platform[0]+"_"+platform[1])
	// dont hard-code references to the AFS rw-volume
	if strings.HasPrefix(goroot, "/afs/.") {
		goroot = strings.Replace(goroot, "/afs/.", "/afs/", 1)
	}
	return create_setup_files(goroot, platform)
}

func create_setup_files(goroot string, platform [2]string) error {
	// create setup.sh
	setup_sh, err := os.Create(
		filepath.Join(goroot, "setup.sh"),
	)
	if err != nil {
		return err
	}
	defer setup_sh.Close()
	_, err = setup_sh.WriteString(fmt.Sprintf(`#!/bin/sh

export GOROOT=%s
export PATH=${GOROOT}/bin:${PATH}
export GOPATH=${HOME}/dev/gocode
export PATH=${GOPATH}/bin:${PATH}
`, goroot))
	if err != nil {
		return err
	}

	// create setup.csh
	setup_csh, err := os.Create(
		filepath.Join(goroot, "setup.csh"),
	)
	if err != nil {
		return err
	}
	defer setup_csh.Close()
	_, err = setup_csh.WriteString(fmt.Sprintf(`
setenv GOROOT %s
setenv PATH ${GOROOT}/bin:${PATH}
setenv GOPATH ${HOME}/dev/gocode
setenv PATH ${GOPATH}/bin:${PATH}
`, goroot))
	if err != nil {
		return err
	}

	return err
}

func main() {
	flag.Parse()
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `install-go installs the go-gc runtime.
Usage:

$ install-go [options] <go-version>
`)
		flag.PrintDefaults()
	}

	if flag.NArg() <= 0 {
		flag.Usage()
		os.Exit(1)
	}

	version := flag.Arg(0)
	if version == "" {
		flag.Usage()
		os.Exit(1)
	}

	platforms := [][2]string{
		{"linux", "amd64"},
		{"linux", "386"},
	}

	type download_fct func(version string, platform [2]string) error
	download := go_download

	switch *g_mode {
	case "curl":
		download = curl_download
	case "go":
		download = go_download
	default:
		fmt.Fprintf(os.Stderr, "**error** invalid download mode (%s)\n", *g_mode)
	}

	all_good := true
	errch := make(chan error, len(platforms))
	for _, plat := range platforms {
		go func(plat [2]string) {
			fmt.Printf(":: installing %s-%s...\n", plat[0], plat[1])
			err := download(version, plat)
			errch <- err
			if err != nil {
				return
			}
			fmt.Printf(":: installing %s-%s... [ok]\n", plat[0], plat[1])
		}(plat)
	}

	for i := 0; i < len(platforms); i++ {
		err := <-errch
		if err != nil {
			all_good = false
			fmt.Fprintf(os.Stderr, "**error** %v\n", err)
		}
		if i == len(platforms)-1 {
			close(errch)
		}
	}

	os.Remove(filepath.Join(*g_odir, "latest"))

	err := os.Symlink(
		version,
		filepath.Join(*g_odir, "latest"),
	)
	if err != nil {
		fmt.Fprintf(os.Stderr, "**error** %v\n", err)
		os.Exit(1)
	}

	if !all_good {
		os.Exit(1)
	}
}

// EOF
<|endoftext|>


<commit_before>// +build windows

package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/StackExchange/wmi"
	"github.com/marpaia/graphite-golang"
)

// Win32_Volume - wmic volume where DriveType=3 list brief
// https://msdn.microsoft.com/en-us/library/aa394515(v=vs.85).aspx
type Win32_Volume struct {
	Capacity   uint64
	DriveType  uint32
	FileSystem string
	FreeSpace  uint64
	Label      *string
	Name       string
}

// Win32_ComputerSystem - wmic ComputerSystem
// https://msdn.microsoft.com/en-us/library/aa394102(v=vs.85).aspx
type Win32_ComputerSystem struct {
	TotalPhysicalMemory uint64
}

// Win32_OperatingSystem - blah blah
// https://msdn.microsoft.com/en-us/library/aa394239(v=vs.85).aspx
type Win32_OperatingSystem struct {
	FreePhysicalMemory     uint64
	FreeVirtualMemory      uint64
	NumberOfProcesses      uint64
	TotalVirtualMemorySize uint64
	TotalVisibleMemorySize uint64
	LastBootUpTime         time.Time
}

// Stat represents stats to send to graphite
type Stat struct {
	Key   string    `json:"key"`
	Value string    `json:"value"`
	DT    time.Time `json:"dt"`
}

// Config - structure for various config attributes
type Config struct {
	computerName    string
	graphiteHost    string
	graphitePort    int
	graphiteMode    string
	graphiteEnabled bool
}

func schedule(what func(), delay time.Duration) chan bool {
	stop := make(chan bool)

	go func() {
		for {
			what()
			select {
			case <-time.After(delay):
			case <-stop:
				return
			}
		}
	}()

	return stop
}

func getOperatingSystemStats() (stats []Stat) {
	var dst []Win32_OperatingSystem

	q := wmi.CreateQuery(&dst, "")
	err := wmi.Query(q, &dst)
	if err != nil {
		log.Fatalf("getOperatingSystemStats: %s", err)
	}
	stats = append(stats, Stat{"mem.physical.free", strconv.FormatUint(dst[0].FreePhysicalMemory, 10), time.Now().UTC()})
	stats = append(stats, Stat{"mem.virtual.free", strconv.FormatUint(dst[0].FreeVirtualMemory, 10), time.Now().UTC()})
	stats = append(stats, Stat{"mem.virtual.total", strconv.FormatUint(dst[0].TotalVirtualMemorySize, 10), time.Now().UTC()})
	stats = append(stats, Stat{"mem.visible.total", strconv.FormatUint(dst[0].TotalVisibleMemorySize, 10), time.Now().UTC()})
	stats = append(stats, Stat{"processes.total", strconv.FormatUint(dst[0].NumberOfProcesses, 10), time.Now().UTC()})
	stats = append(stats, Stat{
		"uptime",
		strconv.FormatUint(uint64(time.Since(dst[0].LastBootUpTime).Seconds()), 10),
		time.Now().UTC(),
	})
	return
}

func getComputerSystemStats() (stats []Stat) {
	var dst []Win32_ComputerSystem

	q := wmi.CreateQuery(&dst, "")
	err := wmi.Query(q, &dst)
	if err != nil {
		log.Fatalf("getComputerSystemStats: %s", err)
	}
	stats = append(stats, Stat{"mem.physical.total", strconv.FormatUint(dst[0].TotalPhysicalMemory, 10), time.Now().UTC()})
	return
}

var nonAlphaNumReg = regexp.MustCompile("[^A-Za-z0-9]+")

func happyDriveName(driveLetter string) string {
	return nonAlphaNumReg.ReplaceAllLiteralString(driveLetter, "")
}

func getDiskStats() (stats []Stat) {
	var dst []Win32_Volume

	q := wmi.CreateQuery(&dst, "Where DriveType=3 and NOT Label='System Reserved'")
	err := wmi.Query(q, &dst)
	if err != nil {
		log.Fatalf("getDiskStats: %s", err)
	}
	for _, v := range dst {
		keyPrefix := fmt.Sprintf("disk.%s", happyDriveName(v.Name))
		percent := int64(100 * (float64(v.FreeSpace) / float64(v.Capacity)))

		stats = append(stats, Stat{fmt.Sprintf("%s.percent_used", keyPrefix), strconv.FormatInt(percent, 10), time.Now().UTC()})
		stats = append(stats, Stat{fmt.Sprintf("%s.free", keyPrefix), strconv.FormatUint(v.FreeSpace, 10), time.Now().UTC()})
		stats = append(stats, Stat{fmt.Sprintf("%s.avail", keyPrefix), strconv.FormatUint(v.Capacity, 10), time.Now().UTC()})
	}

	return
}

type typeperfPair struct {
	regexp       *regexp.Regexp
	template     string
	allowedTotal bool
}

var typeperfRewrites = []typeperfPair{
	typeperfPair{
		regexp:   regexp.MustCompile("^LogicalDisk\\((?P<driveName>[^)]+)\\)\\\\Avg. Disk sec\\/(?P<readWrite>(?:Read|Write))$"),
		template: "disk.${driveName}.disk_${readWrite}_io",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Processor\\(_Total\\)\\\\% Processor Time$"),
		template: "system.cpu",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^System\\\\Threads$"),
		template: "system.threads",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^System\\\\Processor Queue Length$"),
		template: "system.processor_queue_length",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^PhysicalDisk\\([0-9 ]*(?P<driveName>[^)]+)\\)\\\\Avg. Disk (?P<readWrite>(?:Read|Write)) Queue Length$"),
		template: "disk.${driveName}.disk_${readWrite}_queue_length",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Memory\\\\Pages/sec$"),
		template: "mem.pages",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Memory\\\\Pages Input/sec$"),
		template: "mem.pages",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Network Interface\\((?P<ifName>[^)]+)\\)\\\\Bytes (?P<receivedSent>(?:Received|Sent))/sec$"),
		template: "network.${ifName}.bytes.${receivedSent}",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Network Interface\\((?P<ifName>[^)]+)\\)\\\\Packets (?P<receivedSent>(?:Received|Sent)) Unicast/sec$"),
		template: "network.${ifName}.packets.unicast_${receivedSent}",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Network Interface\\((?P<ifName>[^)]+)\\)\\\\Packets (?P<receivedSent>(?:Received|Sent)) Non-Unicast/sec$"),
		template: "network.${ifName}.packets.${receivedSent}",
	},
}

func callTypePerf(fields []string) (stats []Stat) {
	var (
		cmdOut  []byte
		err     error
		headers []string
		records [][]string
	)
	cmdName := "typeperf"
	cmdArgs := []string{
		"-sc",
		"1",
	}
	cmdArgs = append(cmdArgs, fields...)
	if cmdOut, err = exec.Command(cmdName, cmdArgs...).Output(); err != nil {
		log.Fatalf("There was an error running typeperf command [%s %s]: %s", cmdName, cmdArgs, err)
		os.Exit(1)
	}
	var lines []string
	for _, line := range strings.Split(string(cmdOut), "\n") {
		if strings.HasPrefix(line, "\"") {
			lines = append(lines, line)
		}
	}

	r := csv.NewReader(strings.NewReader(strings.Join(lines, "\n")))
	records, err = r.ReadAll()
	if err != nil {
		log.Printf("CmdArgs: [%s], Err: %s, String: %s", cmdArgs, err, string(cmdOut))
		return
	}

	headers, records = records[0], records[1:]
	for headerNum, header := range headers[1:] {
		headerParts := strings.Split(header, "\\")
		headers[headerNum] = strings.Join(headerParts[3:len(headerParts)], "\\")
	}
	for _, row := range records {
	ResultLoop:
		for colNum, col := range row[1:] {
			for _, matchers := range typeperfRewrites {
				if results := matchers.regexp.FindStringSubmatch(headers[colNum]); results != nil {
					var variableMap = make(map[string]string)

					if matchers.regexp.NumSubexp() != 0 {
						for nameNo, name := range matchers.regexp.SubexpNames()[1:] {
							if name == "readWrite" || name == "receivedSent" {
								results[nameNo+1] = strings.ToLower(results[nameNo+1])
							}
							if name == "ifName" {
								results[nameNo+1] = happyDriveName(results[nameNo+1])
							}
							if name == "driveName" {
								/* if there are named matches, then don't allow _Total */
								if results[nameNo+1] == "_Total" {
									continue ResultLoop
								}

								/* If its a hidden drive name, ignore - FIXME */
								if strings.HasPrefix(results[nameNo+1], "HarddiskVolume") {
									continue ResultLoop
								}
								results[nameNo+1] = happyDriveName(results[nameNo+1])
							}
							variableMap[name] = results[nameNo+1]
						}
					}

					keyName := os.Expand(matchers.template, func(s string) string {
						return variableMap[s]
					})
					stats = append(stats, Stat{
						keyName,
						col,
						time.Now().UTC(),
					})
					continue ResultLoop
				}
			}
			log.Printf("NOT FOUND --- %s = %s", headers[colNum], col)

		}
	}
	return
}

func getProcessorStats() (stats []Stat) {
	stats = callTypePerf([]string{
		"Processor(_Total)\\% Processor Time",
	})
	return
}
func getTypePerfStats() (stats []Stat) {
	stats = callTypePerf([]string{
		"LogicalDisk(*)\\Avg. Disk sec/Read",
		"LogicalDisk(*)\\Avg. Disk sec/Write",
		"Network Interface(*)\\Bytes Received/sec",
		"Network Interface(*)\\Bytes Sent/sec",
		"Network Interface(*)\\Packets Received Unicast/sec",
		"Network Interface(*)\\Packets Sent Unicast/sec",
		"Network Interface(*)\\Packets Received Non-Unicast/sec",
		"Network Interface(*)\\Packets Sent Non-Unicast/sec",
		//"Memory\\Available MBytes",
		"Memory\\Pages/sec",
		"Memory\\Pages Input/sec",
		"System\\Processor Queue Length",
		"System\\Threads",
		"PhysicalDisk(*)\\Avg. Disk Write Queue Length",
		"PhysicalDisk(*)\\Avg. Disk Read Queue Length",
	})
	return
}

func getStats() (stats []Stat) {
	stats = append(stats, getTypePerfStats()...)
	stats = append(stats, getDiskStats()...)
	stats = append(stats, getProcessorStats()...)
	stats = append(stats, getComputerSystemStats()...)
	stats = append(stats, getOperatingSystemStats()...)
	return
}

var config = Config{}

func getGraphite(config Config) (g *graphite.Graphite) {
	var err error

	if config.graphiteEnabled {
		g, err = graphite.NewGraphite(config.graphiteHost, config.graphitePort)
		if err != nil {
			log.Fatal("Error connecting to graphite", err)
		}
	} else {
		g = graphite.NewGraphiteNop(config.graphiteHost, config.graphitePort)
	}
	return
}

func main() {
	var stats []Stat
	var g *graphite.Graphite
	var err error
	hostname, _ := os.Hostname()

	flag.StringVar(&config.computerName, "computerName", hostname, "Computer Name")
	flag.BoolVar(&config.graphiteEnabled, "graphite", false, "Enable Graphite")
	flag.StringVar(&config.graphiteHost, "graphiteHost", "localhost", "graphite hostname")
	flag.IntVar(&config.graphitePort, "graphitePort", 2003, "graphite port")
	//flag.StringVar(&config.graphiteMode, "graphiteMode", "tcp", "tcp or udp")
	flag.Parse()

	g = getGraphite(config)

	getStatsInterval := func() {
		newStats := getStats()
		stats = append(stats, newStats...)
	}
	outputStatsInterval := func() {
		var stat Stat
		for len(stats) != 0 {
			stat, stats = stats[0], stats[1:]
			err = g.SendMetric(graphite.NewMetric(
				fmt.Sprintf("%s.%s", config.computerName, stat.Key),
				stat.Value,
				stat.DT.Unix(),
			))
			if err != nil {
				stats = append(stats, stat)
				log.Printf("Error writing to: %s", err)
				g = getGraphite(config)
			}
		}
	}
	/*stopGet := */ schedule(getStatsInterval, 5*time.Second)
	/*stopOutput := */ schedule(outputStatsInterval, 5*time.Millisecond)
	for {
		time.Sleep(100 * time.Second)
	}
}
<commit_msg>Fataf instead of fatal so the error message gets printed<commit_after>// +build windows

package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/StackExchange/wmi"
	"github.com/marpaia/graphite-golang"
)

// Win32_Volume - wmic volume where DriveType=3 list brief
// https://msdn.microsoft.com/en-us/library/aa394515(v=vs.85).aspx
type Win32_Volume struct {
	Capacity   uint64
	DriveType  uint32
	FileSystem string
	FreeSpace  uint64
	Label      *string
	Name       string
}

// Win32_ComputerSystem - wmic ComputerSystem
// https://msdn.microsoft.com/en-us/library/aa394102(v=vs.85).aspx
type Win32_ComputerSystem struct {
	TotalPhysicalMemory uint64
}

// Win32_OperatingSystem - blah blah
// https://msdn.microsoft.com/en-us/library/aa394239(v=vs.85).aspx
type Win32_OperatingSystem struct {
	FreePhysicalMemory     uint64
	FreeVirtualMemory      uint64
	NumberOfProcesses      uint64
	TotalVirtualMemorySize uint64
	TotalVisibleMemorySize uint64
	LastBootUpTime         time.Time
}

// Stat represents stats to send to graphite
type Stat struct {
	Key   string    `json:"key"`
	Value string    `json:"value"`
	DT    time.Time `json:"dt"`
}

// Config - structure for various config attributes
type Config struct {
	computerName    string
	graphiteHost    string
	graphitePort    int
	graphiteMode    string
	graphiteEnabled bool
}

func schedule(what func(), delay time.Duration) chan bool {
	stop := make(chan bool)

	go func() {
		for {
			what()
			select {
			case <-time.After(delay):
			case <-stop:
				return
			}
		}
	}()

	return stop
}

func getOperatingSystemStats() (stats []Stat) {
	var dst []Win32_OperatingSystem

	q := wmi.CreateQuery(&dst, "")
	err := wmi.Query(q, &dst)
	if err != nil {
		log.Fatalf("getOperatingSystemStats: %s", err)
	}
	stats = append(stats, Stat{"mem.physical.free", strconv.FormatUint(dst[0].FreePhysicalMemory, 10), time.Now().UTC()})
	stats = append(stats, Stat{"mem.virtual.free", strconv.FormatUint(dst[0].FreeVirtualMemory, 10), time.Now().UTC()})
	stats = append(stats, Stat{"mem.virtual.total", strconv.FormatUint(dst[0].TotalVirtualMemorySize, 10), time.Now().UTC()})
	stats = append(stats, Stat{"mem.visible.total", strconv.FormatUint(dst[0].TotalVisibleMemorySize, 10), time.Now().UTC()})
	stats = append(stats, Stat{"processes.total", strconv.FormatUint(dst[0].NumberOfProcesses, 10), time.Now().UTC()})
	stats = append(stats, Stat{
		"uptime",
		strconv.FormatUint(uint64(time.Since(dst[0].LastBootUpTime).Seconds()), 10),
		time.Now().UTC(),
	})
	return
}

func getComputerSystemStats() (stats []Stat) {
	var dst []Win32_ComputerSystem

	q := wmi.CreateQuery(&dst, "")
	err := wmi.Query(q, &dst)
	if err != nil {
		log.Fatalf("getComputerSystemStats: %s", err)
	}
	stats = append(stats, Stat{"mem.physical.total", strconv.FormatUint(dst[0].TotalPhysicalMemory, 10), time.Now().UTC()})
	return
}

var nonAlphaNumReg = regexp.MustCompile("[^A-Za-z0-9]+")

func happyDriveName(driveLetter string) string {
	return nonAlphaNumReg.ReplaceAllLiteralString(driveLetter, "")
}

func getDiskStats() (stats []Stat) {
	var dst []Win32_Volume

	q := wmi.CreateQuery(&dst, "Where DriveType=3 and NOT Label='System Reserved'")
	err := wmi.Query(q, &dst)
	if err != nil {
		log.Fatalf("getDiskStats: %s", err)
	}
	for _, v := range dst {
		keyPrefix := fmt.Sprintf("disk.%s", happyDriveName(v.Name))
		percent := int64(100 * (float64(v.FreeSpace) / float64(v.Capacity)))

		stats = append(stats, Stat{fmt.Sprintf("%s.percent_used", keyPrefix), strconv.FormatInt(percent, 10), time.Now().UTC()})
		stats = append(stats, Stat{fmt.Sprintf("%s.free", keyPrefix), strconv.FormatUint(v.FreeSpace, 10), time.Now().UTC()})
		stats = append(stats, Stat{fmt.Sprintf("%s.avail", keyPrefix), strconv.FormatUint(v.Capacity, 10), time.Now().UTC()})
	}

	return
}

type typeperfPair struct {
	regexp       *regexp.Regexp
	template     string
	allowedTotal bool
}

var typeperfRewrites = []typeperfPair{
	typeperfPair{
		regexp:   regexp.MustCompile("^LogicalDisk\\((?P<driveName>[^)]+)\\)\\\\Avg. Disk sec\\/(?P<readWrite>(?:Read|Write))$"),
		template: "disk.${driveName}.disk_${readWrite}_io",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Processor\\(_Total\\)\\\\% Processor Time$"),
		template: "system.cpu",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^System\\\\Threads$"),
		template: "system.threads",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^System\\\\Processor Queue Length$"),
		template: "system.processor_queue_length",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^PhysicalDisk\\([0-9 ]*(?P<driveName>[^)]+)\\)\\\\Avg. Disk (?P<readWrite>(?:Read|Write)) Queue Length$"),
		template: "disk.${driveName}.disk_${readWrite}_queue_length",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Memory\\\\Pages/sec$"),
		template: "mem.pages",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Memory\\\\Pages Input/sec$"),
		template: "mem.pages",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Network Interface\\((?P<ifName>[^)]+)\\)\\\\Bytes (?P<receivedSent>(?:Received|Sent))/sec$"),
		template: "network.${ifName}.bytes.${receivedSent}",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Network Interface\\((?P<ifName>[^)]+)\\)\\\\Packets (?P<receivedSent>(?:Received|Sent)) Unicast/sec$"),
		template: "network.${ifName}.packets.unicast_${receivedSent}",
	},
	typeperfPair{
		regexp:   regexp.MustCompile("^Network Interface\\((?P<ifName>[^)]+)\\)\\\\Packets (?P<receivedSent>(?:Received|Sent)) Non-Unicast/sec$"),
		template: "network.${ifName}.packets.${receivedSent}",
	},
}

func callTypePerf(fields []string) (stats []Stat) {
	var (
		cmdOut  []byte
		err     error
		headers []string
		records [][]string
	)
	cmdName := "typeperf"
	cmdArgs := []string{
		"-sc",
		"1",
	}
	cmdArgs = append(cmdArgs, fields...)
	if cmdOut, err = exec.Command(cmdName, cmdArgs...).Output(); err != nil {
		log.Fatalf("There was an error running typeperf command [%s %s]: %s", cmdName, cmdArgs, err)
		os.Exit(1)
	}
	var lines []string
	for _, line := range strings.Split(string(cmdOut), "\n") {
		if strings.HasPrefix(line, "\"") {
			lines = append(lines, line)
		}
	}

	r := csv.NewReader(strings.NewReader(strings.Join(lines, "\n")))
	records, err = r.ReadAll()
	if err != nil {
		log.Printf("CmdArgs: [%s], Err: %s, String: %s", cmdArgs, err, string(cmdOut))
		return
	}

	headers, records = records[0], records[1:]
	for headerNum, header := range headers[1:] {
		headerParts := strings.Split(header, "\\")
		headers[headerNum] = strings.Join(headerParts[3:len(headerParts)], "\\")
	}
	for _, row := range records {
	ResultLoop:
		for colNum, col := range row[1:] {
			for _, matchers := range typeperfRewrites {
				if results := matchers.regexp.FindStringSubmatch(headers[colNum]); results != nil {
					var variableMap = make(map[string]string)

					if matchers.regexp.NumSubexp() != 0 {
						for nameNo, name := range matchers.regexp.SubexpNames()[1:] {
							if name == "readWrite" || name == "receivedSent" {
								results[nameNo+1] = strings.ToLower(results[nameNo+1])
							}
							if name == "ifName" {
								results[nameNo+1] = happyDriveName(results[nameNo+1])
							}
							if name == "driveName" {
								/* if there are named matches, then don't allow _Total */
								if results[nameNo+1] == "_Total" {
									continue ResultLoop
								}

								/* If its a hidden drive name, ignore - FIXME */
								if strings.HasPrefix(results[nameNo+1], "HarddiskVolume") {
									continue ResultLoop
								}
								results[nameNo+1] = happyDriveName(results[nameNo+1])
							}
							variableMap[name] = results[nameNo+1]
						}
					}

					keyName := os.Expand(matchers.template, func(s string) string {
						return variableMap[s]
					})
					stats = append(stats, Stat{
						keyName,
						col,
						time.Now().UTC(),
					})
					continue ResultLoop
				}
			}
			log.Printf("NOT FOUND --- %s = %s", headers[colNum], col)

		}
	}
	return
}

func getProcessorStats() (stats []Stat) {
	stats = callTypePerf([]string{
		"Processor(_Total)\\% Processor Time",
	})
	return
}
func getTypePerfStats() (stats []Stat) {
	stats = callTypePerf([]string{
		"LogicalDisk(*)\\Avg. Disk sec/Read",
		"LogicalDisk(*)\\Avg. Disk sec/Write",
		"Network Interface(*)\\Bytes Received/sec",
		"Network Interface(*)\\Bytes Sent/sec",
		"Network Interface(*)\\Packets Received Unicast/sec",
		"Network Interface(*)\\Packets Sent Unicast/sec",
		"Network Interface(*)\\Packets Received Non-Unicast/sec",
		"Network Interface(*)\\Packets Sent Non-Unicast/sec",
		//"Memory\\Available MBytes",
		"Memory\\Pages/sec",
		"Memory\\Pages Input/sec",
		"System\\Processor Queue Length",
		"System\\Threads",
		"PhysicalDisk(*)\\Avg. Disk Write Queue Length",
		"PhysicalDisk(*)\\Avg. Disk Read Queue Length",
	})
	return
}

func getStats() (stats []Stat) {
	stats = append(stats, getTypePerfStats()...)
	stats = append(stats, getDiskStats()...)
	stats = append(stats, getProcessorStats()...)
	stats = append(stats, getComputerSystemStats()...)
	stats = append(stats, getOperatingSystemStats()...)
	return
}

var config = Config{}

func getGraphite(config Config) (g *graphite.Graphite) {
	var err error

	if config.graphiteEnabled {
		g, err = graphite.NewGraphite(config.graphiteHost, config.graphitePort)
		if err != nil {
			log.Fatalf("Error connecting to graphite: %s", err)
		}
	} else {
		g = graphite.NewGraphiteNop(config.graphiteHost, config.graphitePort)
	}
	return
}

func main() {
	var stats []Stat
	var g *graphite.Graphite
	var err error
	hostname, _ := os.Hostname()

	flag.StringVar(&config.computerName, "computerName", hostname, "Computer Name")
	flag.BoolVar(&config.graphiteEnabled, "graphite", false, "Enable Graphite")
	flag.StringVar(&config.graphiteHost, "graphiteHost", "localhost", "graphite hostname")
	flag.IntVar(&config.graphitePort, "graphitePort", 2003, "graphite port")
	//flag.StringVar(&config.graphiteMode, "graphiteMode", "tcp", "tcp or udp")
	flag.Parse()

	g = getGraphite(config)

	getStatsInterval := func() {
		newStats := getStats()
		stats = append(stats, newStats...)
	}
	outputStatsInterval := func() {
		var stat Stat
		for len(stats) != 0 {
			stat, stats = stats[0], stats[1:]
			err = g.SendMetric(graphite.NewMetric(
				fmt.Sprintf("%s.%s", config.computerName, stat.Key),
				stat.Value,
				stat.DT.Unix(),
			))
			if err != nil {
				stats = append(stats, stat)
				log.Printf("Error writing to: %s", err)
				g = getGraphite(config)
			}
		}
	}
	/*stopGet := */ schedule(getStatsInterval, 5*time.Second)
	/*stopOutput := */ schedule(outputStatsInterval, 5*time.Millisecond)
	for {
		time.Sleep(100 * time.Second)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"time"

	MQTT "github.com/eclipse/paho.mqtt.golang"
	"github.com/google/uuid"
)

var (
	subTopics = []string{
		"/cooktroller/charles/smoker/set_temp",
		"/cooktroller/charles/smoker/runstate",
	}
	serverState ServerState
)

//Server state and tuning parameters
type ServerState struct {
	KP      float32
	KI      float32
	KD      float32
	Loop    int16
	Running bool
	TempSet float32
}

//Server status message
type ServerStatus struct {
	DeviceID string  `json:"device_id"`
	Running  bool    `json:"running"`
	TempCel  float32 `json:"temp_cel"`
	TempFar  float32 `json:"temp_far"`
}

//SetTemperature unmarshal json to set temp
type SetTemperature struct {
	TempFar float32 `json:"temp_far"`
	TempCel float32 `json:"temp_cel"`
}

//SetRunningState unmarshal json to set run state
type SetRunningState struct {
	Running bool `json:"on"`
}

func main() {
	log.Println("Hello world")
	err := connect()
	if err != nil {
		log.Println("application died")
	}
}

//Connect to and setup MQTT
func connect() error {
	cid := uuid.New().String()
	connOpts := MQTT.NewClientOptions()
	connOpts.SetClientID(cid)
	connOpts.SetCleanSession(true)
	connOpts.SetAutoReconnect(true)
	connOpts.SetMaxReconnectInterval(1 * time.Second)
	connOpts.SetTLSConfig(getTLSConfig())
	log.Println("Certs loaded")
	brokerURL := fmt.Sprintf("tcps://%s:%d%s", "a10cp24047duti.iot.us-east-1.amazonaws.com", 8883, "/")
	connOpts.AddBroker(brokerURL)
	mqttClient := MQTT.NewClient(connOpts)
	token := mqttClient.Connect()
	token.WaitTimeout(30 * time.Second)
	token.Wait()

	if token.Error() != nil {
		log.Println(token.Error())
		return token.Error()
	}
	log.Println("MQTT Endpoint connected")
	for _, topic := range subTopics {
		token := mqttClient.Subscribe(topic, 1, subscriber)
		token.WaitTimeout(30 * time.Second)
		token.Wait()
		if token.Error() != nil {
			return token.Error()
		}
		log.Println("Subscribed to topic: ", topic)
	}
	//Start the PID control loop
	pidControlLoop(10*time.Second, doControl, mqttClient)
	return nil
}

//Function to subscribe to a topic
func subscriber(client MQTT.Client, msg MQTT.Message) {
	log.Println("Message", string(msg.Payload()))
	if msg.Topic() == subTopics[0] {
		var setTemp SetTemperature
		err := json.Unmarshal(msg.Payload(), setTemp)
		if err != nil {
			log.Println("Unable to unmarshal temp set")
			return
		}
		serverState.TempSet = setTemp.TempFar //set the temperature (F for now)
	} else if msg.Topic() == subTopics[1] {
		var setRunState SetRunningState
		err := json.Unmarshal(msg.Payload(), setRunState)
		if err != nil {
			log.Println("Unable to unmarshal set running state")
			return
		}
		serverState.Running = setRunState.Running //set opcode running
	} else {
		log.Println("Message from unknown topic")
	}

}

//Enter a loop to run the PID controller on a schedule
func pidControlLoop(d time.Duration, f func(time.Time), mqttClient MQTT.Client) {
	for x := range time.Tick(d) {
		log.Println("Running control loop")
		go f(x)
		var status ServerStatus
		go status.sendStatus(mqttClient)
	}
}

//TODO: Perform the PID Control
func doControl(t time.Time) {
	log.Println("running control")
}

//Set the status on the struct and send it to the MQTT backend
func (status *ServerStatus) sendStatus(mqttClient MQTT.Client) {
	status.DeviceID = "Cooktroller Smoker"
	status.Running = serverState.Running
	status.TempFar = serverState.TempSet
	data, err := json.Marshal(status)
	if err != nil {
		log.Println(err)
	}
	token := mqttClient.Publish("/cooktroller/charles/smoker/status", 1, false, data)
	token.WaitTimeout(30 * time.Second)
	token.Wait()
	if token.Error() != nil {
		log.Println("Unable to send message to MQTT", token.Error())
	}
}

//NewTlsConfig Load in the certificates and setup the TLS configurations and certs
func getTLSConfig() *tls.Config {
	// Import trusted certificates from CAfile.pem.
	// Alternatively, manually add CA certificates to
	// default openssl CA bundle.
	certpool := x509.NewCertPool()
	pemCerts, err := ioutil.ReadFile("/etc/cooktroller/ca.cert")
	if err == nil {
		certpool.AppendCertsFromPEM(pemCerts)
	}

	// Import client certificate/key pair
	cert, err := tls.LoadX509KeyPair("/etc/cooktroller/cert.pem", "/etc/cooktroller/key.pem")
	if err != nil {
		fmt.Println("Could not load X509 Key pair")
		return nil
	}

	// Just to print out the client certificate..
	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		panic(err)
	}
	//log.Println(cert.Leaf)

	// Create tls.Config with desired tls properties
	return &tls.Config{
		// RootCAs = certs used to verify server cert.
		RootCAs: certpool,
		// ClientAuth = whether to request cert from server.
		// Since the server is set up for SSL, this happens
		// anyways.
		ClientAuth: tls.NoClientCert,
		// ClientCAs = certs used to validate client cert.
		ClientCAs: nil,
		// InsecureSkipVerify = verify that cert contents
		// match server. IP matches what is in cert etc.
		InsecureSkipVerify: true,
		// Certificates = list of certs client sends to server.
		Certificates: []tls.Certificate{cert},
	}
}
<commit_msg>fix: added logging<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"time"

	MQTT "github.com/eclipse/paho.mqtt.golang"
	"github.com/google/uuid"
)

var (
	subTopics = []string{
		"/cooktroller/charles/smoker/set_temp",
		"/cooktroller/charles/smoker/runstate",
	}
	serverState ServerState
)

//Server state and tuning parameters
type ServerState struct {
	KP      float32
	KI      float32
	KD      float32
	Loop    int16
	Running bool
	TempSet float32
}

//Server status message
type ServerStatus struct {
	DeviceID string  `json:"device_id"`
	Running  bool    `json:"running"`
	TempCel  float32 `json:"temp_cel"`
	TempFar  float32 `json:"temp_far"`
}

//SetTemperature unmarshal json to set temp
type SetTemperature struct {
	TempFar float32 `json:"temp_far"`
	TempCel float32 `json:"temp_cel"`
}

//SetRunningState unmarshal json to set run state
type SetRunningState struct {
	Running bool `json:"on"`
}

func main() {
	log.Println("Starting up!")
	err := connect()
	if err != nil {
		log.Println("application died")
	}
}

//Connect to and setup MQTT
func connect() error {
	cid := uuid.New().String()
	connOpts := MQTT.NewClientOptions()
	connOpts.SetClientID(cid)
	connOpts.SetCleanSession(true)
	connOpts.SetAutoReconnect(true)
	connOpts.SetMaxReconnectInterval(1 * time.Second)
	connOpts.SetTLSConfig(getTLSConfig())
	log.Println("Certs loaded")
	brokerURL := fmt.Sprintf("tcps://%s:%d%s", "a10cp24047duti.iot.us-east-1.amazonaws.com", 8883, "/")
	connOpts.AddBroker(brokerURL)
	mqttClient := MQTT.NewClient(connOpts)
	token := mqttClient.Connect()
	token.WaitTimeout(30 * time.Second)
	token.Wait()

	if token.Error() != nil {
		log.Println(token.Error())
		return token.Error()
	}
	log.Println("MQTT Endpoint connected")
	for _, topic := range subTopics {
		token := mqttClient.Subscribe(topic, 1, subscriber)
		token.WaitTimeout(30 * time.Second)
		token.Wait()
		if token.Error() != nil {
			return token.Error()
		}
		log.Println("Subscribed to topic: ", topic)
	}
	//Start the PID control loop
	pidControlLoop(10*time.Second, doControl, mqttClient)
	return nil
}

//Function to subscribe to a topic
func subscriber(client MQTT.Client, msg MQTT.Message) {
	log.Println("Message", string(msg.Payload()))
	if msg.Topic() == subTopics[0] {
		var setTemp SetTemperature
		err := json.Unmarshal(msg.Payload(), setTemp)
		if err != nil {
			log.Println("Unable to unmarshal temp set")
			return
		}
		serverState.TempSet = setTemp.TempFar //set the temperature (F for now)
	} else if msg.Topic() == subTopics[1] {
		var setRunState SetRunningState
		err := json.Unmarshal(msg.Payload(), setRunState)
		if err != nil {
			log.Println("Unable to unmarshal set running state")
			return
		}
		serverState.Running = setRunState.Running //set opcode running
	} else {
		log.Println("Message from unknown topic")
	}

}

//Enter a loop to run the PID controller on a schedule
func pidControlLoop(d time.Duration, f func(time.Time), mqttClient MQTT.Client) {
	for x := range time.Tick(d) {
		log.Println("Running control loop")
		go f(x)
		var status ServerStatus
		go status.sendStatus(mqttClient)
	}
}

//TODO: Perform the PID Control
func doControl(t time.Time) {
	log.Println("running control")
}

//Set the status on the struct and send it to the MQTT backend
func (status *ServerStatus) sendStatus(mqttClient MQTT.Client) {
	status.DeviceID = "Cooktroller Smoker"
	status.Running = serverState.Running
	status.TempFar = serverState.TempSet
	data, err := json.Marshal(status)
	if err != nil {
		log.Println(err)
	}
	token := mqttClient.Publish("/cooktroller/charles/smoker/status", 1, false, data)
	token.WaitTimeout(30 * time.Second)
	token.Wait()
	if token.Error() != nil {
		log.Println("Unable to send message to MQTT", token.Error())
	}
}

//NewTlsConfig Load in the certificates and setup the TLS configurations and certs
func getTLSConfig() *tls.Config {
	// Import trusted certificates from CAfile.pem.
	// Alternatively, manually add CA certificates to
	// default openssl CA bundle.
	certpool := x509.NewCertPool()
	pemCerts, err := ioutil.ReadFile("/etc/cooktroller/ca.cert")
	if err == nil {
		certpool.AppendCertsFromPEM(pemCerts)
	}

	// Import client certificate/key pair
	cert, err := tls.LoadX509KeyPair("/etc/cooktroller/cert.pem", "/etc/cooktroller/key.pem")
	if err != nil {
		fmt.Println("Could not load X509 Key pair")
		return nil
	}

	// Just to print out the client certificate..
	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		panic(err)
	}
	//log.Println(cert.Leaf)

	// Create tls.Config with desired tls properties
	return &tls.Config{
		// RootCAs = certs used to verify server cert.
		RootCAs: certpool,
		// ClientAuth = whether to request cert from server.
		// Since the server is set up for SSL, this happens
		// anyways.
		ClientAuth: tls.NoClientCert,
		// ClientCAs = certs used to validate client cert.
		ClientCAs: nil,
		// InsecureSkipVerify = verify that cert contents
		// match server. IP matches what is in cert etc.
		InsecureSkipVerify: true,
		// Certificates = list of certs client sends to server.
		Certificates: []tls.Certificate{cert},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"crypto/tls"
	"flag"
	"fmt"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"syscall"
	"time"
)

// hardcoded program settings
const baseDir string = "muck"
const inFile string = "in"
const outFile string = "out"
const timeString string = "2006-01-02T150405"

// Program level switches set from command line
var debugMode bool
var disableLogRotate bool

// MuckServer stores all connection settings
type MuckServer struct {
	name     string
	host     string
	port     uint
	ssl      bool
	insecure bool
}

// Simplify returning connection strings by making it the String method
func (m *MuckServer) String() string {
	s := fmt.Sprintf("%s:%d", m.host, m.port)
	return s
}

func debugLog(log ...interface{}) {
	if debugMode {
		fmt.Print("DEBUG: ")
		fmt.Println(log...)
	}
}

func checkError(err error) {
	if err != nil {
		debugLog("checkError caught", err.Error())
		panic(err.Error())
	}
}

func getTimestamp() string {
	return time.Now().Format(timeString)
}

func initArgs() MuckServer {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "    %v [<flags>] <name> <server> <port>\n", os.Args[0])
		flag.PrintDefaults()
	}

	// Global program flags
	flag.BoolVar(&debugMode, "debug", false, "Enable debug")
	flag.BoolVar(&disableLogRotate, "nolog", false, "Disable log rotation on quit")
	// Connection flags
	ssl := flag.Bool("ssl", false, "Enable ssl")
	insecure := flag.Bool("insecure", false, "Disable strict SSL checking")
	flag.Parse()

	if flag.NArg() != 3 {
		flag.Usage()
		os.Exit(1)
	}

	args := flag.Args()
	p, err := strconv.Atoi(args[2])
	checkError(err)
	s := MuckServer{name: args[0], host: args[1], port: uint(p), ssl: *ssl, insecure: *insecure}

	debugLog("rotate log disabled?:", disableLogRotate)
	debugLog("name:", s.name)
	debugLog("host:", s.host)
	debugLog("port:", s.port)
	debugLog("SSL?:", s.ssl)
	debugLog("insecure ssl check?:", s.insecure)

	return s
}

func getWorkingDir(main string, sub string) string {
	u, err := user.Current()
	checkError(err)
	h := u.HomeDir
	debugLog("home directory", h)

	w := filepath.Join(h, main, sub)
	debugLog("working directory", w)
	return w
}

func makeFIFO(file string) *os.File {
	if _, err := os.Stat(file); err == nil {
		fmt.Println("FIFO already exists. Unlink or exit")
		fmt.Println("If you run multiple connection with the same name you're gonna have a bad time")
		fmt.Print("Type YES to unlink and recreate: ")
		i := bufio.NewReader(os.Stdin)
		a, err := i.ReadString('\n')
		checkError(err)
		if a != "YES\n" {
			fmt.Println("Canceling. Please remove FIFO before running")
			panic("User Canceled at FIFO removal prompt")
		}
		errUn := syscall.Unlink(file)
		checkError(errUn)
		debugLog(file, "unlinked")
	}
	err := syscall.Mkfifo(file, 0644)
	checkError(err)
	debugLog("FIFO created as", file)
	f, err := os.OpenFile(file, os.O_RDONLY|syscall.O_NONBLOCK, 0666)
	checkError(err)
	debugLog("FIFO opened as", f.Name())
	return f
}

func makeOut(file string) *os.File {
	if _, err := os.Stat(file); err == nil {
		fmt.Printf("Warning: %v already exists; appending.\n", file)
	}
	out, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	checkError(err)
	debugLog("logfile created as", out.Name())
	return out
}

func lookupHostname(s string) *net.TCPAddr {
	a, err := net.ResolveTCPAddr("tcp", s)
	checkError(err)
	debugLog("server resolves to", a)
	return a
}

func setupConnection(s *MuckServer) net.Conn {
	tcpAddr := lookupHostname(s.String())
	connection, err := net.DialTCP("tcp", nil, tcpAddr)
	checkError(err)
	debugLog("connected to Server")

	// We keep alive for mucks
	errSka := connection.SetKeepAlive(true)
	checkError(errSka)
	keepalive := 15 * time.Minute
	errSkap := connection.SetKeepAlivePeriod(keepalive)
	checkError(errSkap)
	return connection
}

func setupTLSConnextion(s *MuckServer) net.Conn {
	var conf *tls.Config
	if s.insecure {
		conf = &tls.Config{InsecureSkipVerify: true}
	} else {
		conf = &tls.Config{ServerName: s.host}
	}
	tcpAddr := lookupHostname(s.String())
	connection, err := tls.Dial("tcp", tcpAddr.String(), conf)
	checkError(err)
	return connection
}

func readtoConn(f *os.File, c net.Conn, quit chan bool) {
	tmpError := fmt.Sprintf("read %v: resource temporarily unavailable", f.Name())
	for {
		select {
		case <-quit:
			debugLog("readtoConn recieved quit; returning")
			return
		default:
			buf := make([]byte, 512)
			bi, err := f.Read(buf)
			if err != nil && err.Error() != "EOF" && err.Error() != tmpError {
				checkError(err)
			} else if bi == 0 {
				continue
			}
			debugLog(bi, "bytes read from FIFO")
			bo, err := c.Write(buf[:bi])
			checkError(err)
			debugLog(bo, "bytes written to file")
		}
	}
}

func readToFile(c net.Conn, f *os.File, quit chan bool) {
	_, err := f.WriteString(fmt.Sprintf("~Connected at %v\n", getTimestamp()))
	checkError(err)
	for {
		buf := make([]byte, 512)
		bi, err := c.Read(buf)
		if err != nil {
			fmt.Println("Server disconnected with", err.Error())
			_, err := f.WriteString(fmt.Sprintf("\n~Connection lost at %v\n", getTimestamp()))
			checkError(err)
			quit <- true
			return
		}
		debugLog(bi, "bytes read from connection")

		bo, err := f.Write(buf[:bi])
		checkError(err)
		debugLog(bo, "bytes written to file")
	}
}

func closeConnection(c net.Conn) {
	err := c.Close()
	if err != nil {
		debugLog(err.Error())
	}
	debugLog("connection closed")
}

func closeFIFO(f *os.File) {
	n := f.Name()
	debugLog("closing and deleting FIFO", n)
	errC := f.Close()
	if errC != nil {
		debugLog(errC.Error())
	}
	errU := syscall.Unlink(n)
	if errU != nil {
		debugLog(errU.Error())
	}
	debugLog(n, "closed and deleted")
}

func closeLog(f *os.File) {
	n := f.Name()
	debugLog("closing and rotating file", n)
	errC := f.Close()
	if errC != nil {
		debugLog(errC.Error())
	}
	debugLog(n, "closed")
	if disableLogRotate {
		debugLog("log rotation is disabled")
		return
	}
	errR := os.Rename(outFile, getTimestamp())
	if errR != nil {
		debugLog(errR.Error())
	}
	debugLog(n, "rotated")
}

func main() {
	// checkError throws a panic, catch it at the end and return error to user
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("FATAL ERROR", r)
		}
	}()

	fmt.Println("Started at", getTimestamp())
	server := initArgs()

	// Make and move to working directory
	workingDir := getWorkingDir(baseDir, server.name)
	errMk := os.MkdirAll(workingDir, 0755)
	checkError(errMk)

	errCh := os.Chdir(workingDir)
	checkError(errCh)

	// Make the in FIFO
	in := makeFIFO(inFile)
	defer closeFIFO(in)

	// Make the out file
	out := makeOut(outFile)
	defer closeLog(out)

	//create connection
	var connection net.Conn
	if server.ssl {
		connection = setupTLSConnextion(&server)
	} else {
		connection = setupConnection(&server)
	}
	defer closeConnection(connection)

	quit := make(chan bool)
	go readToFile(connection, out, quit)
	readtoConn(in, connection, quit)

	fmt.Println("Quit at", getTimestamp())
	fmt.Println("Thanks for playing!")
	debugLog("end of main hit")
}
<commit_msg>Add sleep to FIFO read to fix errors and CPU usage<commit_after>package main

import (
	"bufio"
	"crypto/tls"
	"flag"
	"fmt"
	"net"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"syscall"
	"time"
)

// hardcoded program settings
const baseDir string = "muck"
const inFile string = "in"
const outFile string = "out"
const timeString string = "2006-01-02T150405"
const bufferSize int = 1024

// Program level switches set from command line
var debugMode bool
var disableLogRotate bool

// MuckServer stores all connection settings
type MuckServer struct {
	name     string
	host     string
	port     uint
	ssl      bool
	insecure bool
}

// Simplify returning connection strings by making it the String method
func (m *MuckServer) String() string {
	s := fmt.Sprintf("%s:%d", m.host, m.port)
	return s
}

func debugLog(log ...interface{}) {
	if debugMode {
		fmt.Print("DEBUG: ")
		fmt.Println(log...)
	}
}

func checkError(err error) {
	if err != nil {
		debugLog("checkError caught", err.Error())
		panic(err.Error())
	}
}

func getTimestamp() string {
	return time.Now().Format(timeString)
}

func initArgs() MuckServer {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "    %v [<flags>] <name> <server> <port>\n", os.Args[0])
		flag.PrintDefaults()
	}

	// Global program flags
	flag.BoolVar(&debugMode, "debug", false, "Enable debug")
	flag.BoolVar(&disableLogRotate, "nolog", false, "Disable log rotation on quit")
	// Connection flags
	ssl := flag.Bool("ssl", false, "Enable ssl")
	insecure := flag.Bool("insecure", false, "Disable strict SSL checking")
	flag.Parse()

	if flag.NArg() != 3 {
		flag.Usage()
		os.Exit(1)
	}

	args := flag.Args()
	p, err := strconv.Atoi(args[2])
	checkError(err)
	s := MuckServer{name: args[0], host: args[1], port: uint(p), ssl: *ssl, insecure: *insecure}

	debugLog("rotate log disabled?:", disableLogRotate)
	debugLog("name:", s.name)
	debugLog("host:", s.host)
	debugLog("port:", s.port)
	debugLog("SSL?:", s.ssl)
	debugLog("insecure ssl check?:", s.insecure)

	return s
}

func getWorkingDir(main string, sub string) string {
	u, err := user.Current()
	checkError(err)
	h := u.HomeDir
	debugLog("home directory", h)

	w := filepath.Join(h, main, sub)
	debugLog("working directory", w)
	return w
}

func makeFIFO(file string) *os.File {
	if _, err := os.Stat(file); err == nil {
		fmt.Println("FIFO already exists. Unlink or exit")
		fmt.Println("If you run multiple connection with the same name you're gonna have a bad time")
		fmt.Print("Type YES to unlink and recreate: ")
		i := bufio.NewReader(os.Stdin)
		a, err := i.ReadString('\n')
		checkError(err)
		if a != "YES\n" {
			fmt.Println("Canceling. Please remove FIFO before running")
			panic("User Canceled at FIFO removal prompt")
		}
		errUn := syscall.Unlink(file)
		checkError(errUn)
		debugLog(file, "unlinked")
	}
	err := syscall.Mkfifo(file, 0644)
	checkError(err)
	debugLog("FIFO created as", file)
	f, err := os.OpenFile(file, os.O_RDONLY|syscall.O_NONBLOCK, 0666)
	checkError(err)
	debugLog("FIFO opened as", f.Name())
	return f
}

func makeOut(file string) *os.File {
	if _, err := os.Stat(file); err == nil {
		fmt.Printf("Warning: %v already exists; appending.\n", file)
	}
	out, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	checkError(err)
	debugLog("logfile created as", out.Name())
	return out
}

func lookupHostname(s string) *net.TCPAddr {
	a, err := net.ResolveTCPAddr("tcp", s)
	checkError(err)
	debugLog("server resolves to", a)
	return a
}

func setupConnection(s *MuckServer) net.Conn {
	tcpAddr := lookupHostname(s.String())
	connection, err := net.DialTCP("tcp", nil, tcpAddr)
	checkError(err)
	debugLog("connected to Server")

	// We keep alive for mucks
	errSka := connection.SetKeepAlive(true)
	checkError(errSka)
	keepalive := 15 * time.Minute
	errSkap := connection.SetKeepAlivePeriod(keepalive)
	checkError(errSkap)
	return connection
}

func setupTLSConnextion(s *MuckServer) net.Conn {
	var conf *tls.Config
	if s.insecure {
		conf = &tls.Config{InsecureSkipVerify: true}
	} else {
		conf = &tls.Config{ServerName: s.host}
	}
	tcpAddr := lookupHostname(s.String())
	connection, err := tls.Dial("tcp", tcpAddr.String(), conf)
	checkError(err)
	return connection
}

func readToConn(f *os.File, c net.Conn, quit chan bool) {
	for {
		select {
		case <-quit:
			debugLog("readtoConn recieved quit; returning")
			return
		default:
			// This pause between reads from the FIFO. This is the difference between
			// 0% and 100% cpu usage. Also without this you will get "read %v:
			// resource temporarily unavailable" errors
			time.Sleep(time.Second / 10)
			buf := make([]byte, bufferSize)
			bi, err := f.Read(buf)
			if err != nil && err.Error() != "EOF" {
				checkError(err)
			} else if bi == 0 {
				time.Sleep(time.Second)
				continue
			}
			debugLog(bi, "bytes read from FIFO")
			bo, err := c.Write(buf[:bi])
			checkError(err)
			debugLog(bo, "bytes written to file")
		}
	}
}

func readToFile(c net.Conn, f *os.File, quit chan bool) {
	_, err := f.WriteString(fmt.Sprintf("~Connected at %v\n", getTimestamp()))
	checkError(err)
	for {
		buf := make([]byte, bufferSize)
		bi, err := c.Read(buf)
		if err != nil {
			fmt.Println("Server disconnected with", err.Error())
			_, err := f.WriteString(fmt.Sprintf("\n~Connection lost at %v\n", getTimestamp()))
			checkError(err)
			quit <- true
			return
		}
		debugLog(bi, "bytes read from connection")

		bo, err := f.Write(buf[:bi])
		checkError(err)
		debugLog(bo, "bytes written to file")
	}
}

func closeConnection(c net.Conn) {
	err := c.Close()
	if err != nil {
		debugLog(err.Error())
	}
	debugLog("connection closed")
}

func closeFIFO(f *os.File) {
	n := f.Name()
	debugLog("closing and deleting FIFO", n)
	errC := f.Close()
	if errC != nil {
		debugLog(errC.Error())
	}
	errU := syscall.Unlink(n)
	if errU != nil {
		debugLog(errU.Error())
	}
	debugLog(n, "closed and deleted")
}

func closeLog(f *os.File) {
	n := f.Name()
	debugLog("closing and rotating file", n)
	errC := f.Close()
	if errC != nil {
		debugLog(errC.Error())
	}
	debugLog(n, "closed")
	if disableLogRotate {
		debugLog("log rotation is disabled")
		return
	}
	errR := os.Rename(outFile, getTimestamp())
	if errR != nil {
		debugLog(errR.Error())
	}
	debugLog(n, "rotated")
}

func main() {
	// checkError throws a panic, catch it at the end and return error to user
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("FATAL ERROR", r)
		}
	}()

	fmt.Println("Started at", getTimestamp())
	server := initArgs()

	// Make and move to working directory
	workingDir := getWorkingDir(baseDir, server.name)
	errMk := os.MkdirAll(workingDir, 0755)
	checkError(errMk)

	errCh := os.Chdir(workingDir)
	checkError(errCh)

	// Make the in FIFO
	in := makeFIFO(inFile)
	defer closeFIFO(in)

	// Make the out file
	out := makeOut(outFile)
	defer closeLog(out)

	//create connection
	var connection net.Conn
	if server.ssl {
		connection = setupTLSConnextion(&server)
	} else {
		connection = setupConnection(&server)
	}
	defer closeConnection(connection)

	quit := make(chan bool)
	go readToFile(connection, out, quit)
	readToConn(in, connection, quit)

	fmt.Println("Quit at", getTimestamp())
	fmt.Println("Thanks for playing!")
	debugLog("end of main hit")
}
<|endoftext|>