<commit_before>package workers

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"
)

const (
	_STORAGE_REDIS  = "redis"
	_STORAGE_MYSQL  = "mysql"
	_STORAGE_ORACLE = "oracle"
)

type StorageOption struct {
	Type  string
	Db    string
	Table string //当Type为mysql/oracle有用
	Host  string
	Port  string
	Pwd   string
}

type LocalStorage struct {
	option *StorageOption // 存储选项
	key    string
	value  string
	expire int
	ts     int
}

/* {{{ func (w *OmqWorker) localStorage(cmd []string) error
 * 处理SET/DEL命令
 */
func (w *OmqWorker) localStorage(cmd []string) error { //set + del

	if Redis == nil { //没有本地存储
		return fmt.Errorf("can't reach localstorage")
	} else {
		w.Trace("save local storage: %q", cmd)
	}

	// 解析命令
	if len(cmd) >= 4 {
		act := strings.ToUpper(cmd[0])
		ls := new(LocalStorage)
		option := cmd[1]
		ls.key = cmd[2]
		if option == "" || strings.ToLower(option) == "redis" {
			//兼容旧版, 新版应该传入一个json,或者为空
			ls.option = &StorageOption{Type: _STORAGE_REDIS}
		} else {
			//解析
			o := new(StorageOption)
			if err := json.Unmarshal([]byte(option), o); err != nil {
				w.Info("unmarshal option failed: %s", option)
				ls.option = &StorageOption{Type: _STORAGE_REDIS} //默认
			} else {
				ls.option = o
			}
		}
		ls.value = cmd[3]
		w.Info("[act: %s][key: %s][value: %s][expire: %d]", act, ls.key, ls.value, ls.expire)
		switch act {
		case COMMAND_SET:
			if len(cmd) >= 5 {
				ls.expire, _ = strconv.Atoi(cmd[4])
			}
			return ls.Set()
		case COMMAND_DEL:
			return ls.Del()
		case COMMAND_SCHEDULE:
			if len(cmd) >= 5 {
				ls.ts, _ = strconv.Atoi(cmd[4])
			}
			return ls.Schedule()
		default:
			return fmt.Errorf("action error: %s", act)
		}
	} else {
		return fmt.Errorf("command error: %s", cmd)
	}

	return nil
}

/* }}} */

/* {{{ func (w *OmqWorker) localGet(cmd []string) ([]string, error)
 * 处理SET/DEL命令
 */
func (w *OmqWorker) localGet(cmd []string) (r []string, err error) { //set + del

	if Redis == nil { //没有本地存储
		err = fmt.Errorf("can't reach localstorage")
		return
	} else {
		w.Trace("save local storage: %q", cmd)
	}

	// 解析命令
	if len(cmd) >= 3 {
		act := strings.ToUpper(cmd[0])
		ls := new(LocalStorage)
		option := cmd[1]
		ls.key = cmd[2]
		if option == "" || strings.ToLower(option) == "redis" {
			//兼容旧版, 新版应该传入一个json,或者为空
			ls.option = &StorageOption{Type: _STORAGE_REDIS}
		} else {
			//解析
			o := new(StorageOption)
			if err := json.Unmarshal([]byte(option), o); err != nil {
				w.Info("unmarshal option failed: %s", option)
				ls.option = &StorageOption{Type: _STORAGE_REDIS} //默认
			} else {
				ls.option = o
			}
		}
		w.Info("[act: %s][key: %s]", act, ls.key)
		switch act {
		case COMMAND_GET:
			return ls.Get()
		case COMMAND_TIMING:
			return ls.Timing()
		default:
			return nil, fmt.Errorf("action error: %s", act)
		}
	} else {
		err = fmt.Errorf("command error: %s", cmd)
	}

	return
}

/* }}} */

func (ls *LocalStorage) Set() (err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	if ls.expire > 0 {
		_, err = redisConn.Do("SET", ls.key, ls.value, "EX", ls.expire)
	} else {
		_, err = redisConn.Do("SET", ls.key, ls.value)
	}
	return
}

func (ls *LocalStorage) Get() (r []string, err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	if result, e := redisConn.Do("GET", ls.key); e == nil {
		if result == nil {
			r = nil
		} else {
			r = []string{string(result.([]byte))}
		}
	} else {
		err = e
	}
	return
}

func (ls *LocalStorage) Timing() (r []string, err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	now := time.Now().Unix() //当前的时间戳
	//if result, e := redisConn.Do("ZRANGEBYSCORE", ls.key, 0, now, "LIMIT", 0, 1); e == nil {
	if result, e := redisConn.Do("ZRANGEBYSCORE", ls.key, 0, now, "WITHSCORES", "LIMIT", 0, 10); e == nil {
		if result != nil {
			switch rt := result.(type) {
			case []byte:
				if len(rt) == 0 {
					return nil, ErrNil
				}
				r = []string{string(rt)}
			case []interface{}:
				if len(rt) == 0 {
					return nil, ErrNil
				}
				r = make([]string, 0)
				for _, rtt := range rt {
					r = append(r, string(rtt.([]byte)))
				}
			default:
				err = fmt.Errorf("unknown type")
			}
		}
	} else {
		err = e
	}
	return
}

func (ls *LocalStorage) Del() error {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	_, err := redisConn.Do("DEL", ls.key)
	return err
}

func (ls *LocalStorage) Schedule() (err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	_, err = redisConn.Do("ZADD", ls.key, ls.ts, ls.value)
	return
}
<commit_msg>timing auto remove member<commit_after>package workers

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"
)

const (
	_STORAGE_REDIS  = "redis"
	_STORAGE_MYSQL  = "mysql"
	_STORAGE_ORACLE = "oracle"
)

type StorageOption struct {
	Type  string
	Db    string
	Table string //当Type为mysql/oracle有用
	Host  string
	Port  string
	Pwd   string
}

type LocalStorage struct {
	option *StorageOption // 存储选项
	key    string
	value  string
	expire int
	ts     int
}

/* {{{ func (w *OmqWorker) localStorage(cmd []string) error
 * 处理SET/DEL命令
 */
func (w *OmqWorker) localStorage(cmd []string) error { //set + del

	if Redis == nil { //没有本地存储
		return fmt.Errorf("can't reach localstorage")
	} else {
		w.Trace("save local storage: %q", cmd)
	}

	// 解析命令
	if len(cmd) >= 4 {
		act := strings.ToUpper(cmd[0])
		ls := new(LocalStorage)
		option := cmd[1]
		ls.key = cmd[2]
		if option == "" || strings.ToLower(option) == "redis" {
			//兼容旧版, 新版应该传入一个json,或者为空
			ls.option = &StorageOption{Type: _STORAGE_REDIS}
		} else {
			//解析
			o := new(StorageOption)
			if err := json.Unmarshal([]byte(option), o); err != nil {
				w.Info("unmarshal option failed: %s", option)
				ls.option = &StorageOption{Type: _STORAGE_REDIS} //默认
			} else {
				ls.option = o
			}
		}
		ls.value = cmd[3]
		w.Info("[act: %s][key: %s][value: %s][expire: %d]", act, ls.key, ls.value, ls.expire)
		switch act {
		case COMMAND_SET:
			if len(cmd) >= 5 {
				ls.expire, _ = strconv.Atoi(cmd[4])
			}
			return ls.Set()
		case COMMAND_DEL:
			return ls.Del()
		case COMMAND_SCHEDULE:
			if len(cmd) >= 5 {
				ls.ts, _ = strconv.Atoi(cmd[4])
			}
			return ls.Schedule()
		default:
			return fmt.Errorf("action error: %s", act)
		}
	} else {
		return fmt.Errorf("command error: %s", cmd)
	}

	return nil
}

/* }}} */

/* {{{ func (w *OmqWorker) localGet(cmd []string) ([]string, error)
 * 处理SET/DEL命令
 */
func (w *OmqWorker) localGet(cmd []string) (r []string, err error) { //set + del

	if Redis == nil { //没有本地存储
		err = fmt.Errorf("can't reach localstorage")
		return
	} else {
		w.Trace("save local storage: %q", cmd)
	}

	// 解析命令
	if len(cmd) >= 3 {
		act := strings.ToUpper(cmd[0])
		ls := new(LocalStorage)
		option := cmd[1]
		ls.key = cmd[2]
		if option == "" || strings.ToLower(option) == "redis" {
			//兼容旧版, 新版应该传入一个json,或者为空
			ls.option = &StorageOption{Type: _STORAGE_REDIS}
		} else {
			//解析
			o := new(StorageOption)
			if err := json.Unmarshal([]byte(option), o); err != nil {
				w.Info("unmarshal option failed: %s", option)
				ls.option = &StorageOption{Type: _STORAGE_REDIS} //默认
			} else {
				ls.option = o
			}
		}
		w.Info("[act: %s][key: %s]", act, ls.key)
		switch act {
		case COMMAND_GET:
			return ls.Get()
		case COMMAND_TIMING:
			return ls.Timing()
		default:
			return nil, fmt.Errorf("action error: %s", act)
		}
	} else {
		err = fmt.Errorf("command error: %s", cmd)
	}

	return
}

/* }}} */

func (ls *LocalStorage) Set() (err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	if ls.expire > 0 {
		_, err = redisConn.Do("SET", ls.key, ls.value, "EX", ls.expire)
	} else {
		_, err = redisConn.Do("SET", ls.key, ls.value)
	}
	return
}

func (ls *LocalStorage) Get() (r []string, err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	if result, e := redisConn.Do("GET", ls.key); e == nil {
		if result == nil {
			r = nil
		} else {
			r = []string{string(result.([]byte))}
		}
	} else {
		err = e
	}
	return
}

func (ls *LocalStorage) Timing() (r []string, err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	now := time.Now().Unix() //当前的时间戳
	//if result, e := redisConn.Do("ZRANGEBYSCORE", ls.key, 0, now, "LIMIT", 0, 1); e == nil {
	if result, e := redisConn.Do("ZRANGEBYSCORE", ls.key, 0, now, "WITHSCORES", "LIMIT", 0, 10); e == nil {
		if result != nil {
			switch rt := result.(type) {
			case []byte:
				if len(rt) == 0 {
					return nil, ErrNil
				}
				r = []string{string(rt)}
			case []interface{}:
				if len(rt) == 0 {
					return nil, ErrNil
				}
				//zrem := make([]string, 0)
				r = make([]string, 0)
				for k, rtt := range rt {
					v := string(rtt.([]byte))
					r = append(r, v)
					if k%2 == 0 { //偶数为member(0-based)
						//fmt.Printf("k:%d,v:%s\n", k, v)
						//zrem = append(zrem, v)
						redisConn.Do("ZREM", ls.key, v)
					}
				}
				//redisConn.Do("ZREM", ls.key, strings.Join(zrem, " "))
				//redisConn.Do("ZREM", ls.key, zrem)
			default:
				err = fmt.Errorf("unknown type")
			}
		}
	} else {
		err = e
	}
	return
}

func (ls *LocalStorage) Del() error {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	_, err := redisConn.Do("DEL", ls.key)
	return err
}

func (ls *LocalStorage) Schedule() (err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	_, err = redisConn.Do("ZADD", ls.key, ls.ts, ls.value)
	return
}
<|endoftext|>


<commit_before>package workers

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

const (
	_STORAGE_REDIS  = "redis"
	_STORAGE_MYSQL  = "mysql"
	_STORAGE_ORACLE = "oracle"
)

type StorageOption struct {
	Type  string
	Db    string
	Table string //当Type为mysql/oracle有用
	Host  string
	Port  string
	Pwd   string
}

type LocalStorage struct {
	option *StorageOption // 存储选项
	key    string
	value  string
	expire int
}

func (w *OmqWorker) localStorage(cmd []string) error {

	if Redis == nil { //没有本地存储
		return fmt.Errorf("can't reach localstorage")
	} else {
		w.Trace("save local storage: %q", cmd)
	}

	// 解析命令
	if len(cmd) >= 3 {
		act := strings.ToLower(cmd[0])
		ls := new(LocalStorage)
		option := cmd[1]
		ls.key = cmd[2]
		ls.value = cmd[3]
		if option == "" || strings.ToLower(option) == "redis" {
			//兼容旧版, 新版应该传入一个json,或者为空
			ls.option = &StorageOption{Type: _STORAGE_REDIS}
		} else {
			//解析
			o := new(StorageOption)
			if err := json.Unmarshal([]byte(option), o); err != nil {
				w.Info("unmarshal option failed: %s", option)
				ls.option = &StorageOption{Type: _STORAGE_REDIS} //默认
			} else {
				ls.option = o
			}
		}
		if len(cmd) >= 5 {
			ls.expire, _ = strconv.Atoi(cmd[3])
		}
		w.Info("[act: %s][key: %s][value: %s][expire: %d]", act, ls.key, ls.value, ls.expire)
		switch act {
		case "set":
			return ls.Set()
		case "get":
			return ls.Get()
		case "del":
			return ls.Del()
		default:
			return fmt.Errorf("action error: %s", act)
		}
	} else {
		return fmt.Errorf("command error: %s", cmd)
	}

	return nil
}

func (ls *LocalStorage) Set() (err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	if ls.expire > 0 {
		_, err = redisConn.Do("SET", ls.key, ls.value, "EX", ls.expire)
	} else {
		_, err = redisConn.Do("SET", ls.key, ls.value)
	}
	return
}

func (ls *LocalStorage) Get() error {
	return nil
}
func (ls *LocalStorage) Del() error {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	_, err := redisConn.Do("DEL", ls.key)
	return err
}
<commit_msg>bugfix<commit_after>package workers

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
)

const (
	_STORAGE_REDIS  = "redis"
	_STORAGE_MYSQL  = "mysql"
	_STORAGE_ORACLE = "oracle"
)

type StorageOption struct {
	Type  string
	Db    string
	Table string //当Type为mysql/oracle有用
	Host  string
	Port  string
	Pwd   string
}

type LocalStorage struct {
	option *StorageOption // 存储选项
	key    string
	value  string
	expire int
}

func (w *OmqWorker) localStorage(cmd []string) error {

	if Redis == nil { //没有本地存储
		return fmt.Errorf("can't reach localstorage")
	} else {
		w.Trace("save local storage: %q", cmd)
	}

	// 解析命令
	if len(cmd) >= 3 {
		act := strings.ToLower(cmd[0])
		ls := new(LocalStorage)
		option := cmd[1]
		ls.key = cmd[2]
		if option == "" || strings.ToLower(option) == "redis" {
			//兼容旧版, 新版应该传入一个json,或者为空
			ls.option = &StorageOption{Type: _STORAGE_REDIS}
		} else {
			//解析
			o := new(StorageOption)
			if err := json.Unmarshal([]byte(option), o); err != nil {
				w.Info("unmarshal option failed: %s", option)
				ls.option = &StorageOption{Type: _STORAGE_REDIS} //默认
			} else {
				ls.option = o
			}
		}
		if len(cmd) >= 4 {
			ls.value = cmd[3]
		}
		if len(cmd) >= 5 {
			ls.expire, _ = strconv.Atoi(cmd[3])
		}
		w.Info("[act: %s][key: %s][value: %s][expire: %d]", act, ls.key, ls.value, ls.expire)
		switch act {
		case "set":
			return ls.Set()
		case "get":
			return ls.Get()
		case "del":
			return ls.Del()
		default:
			return fmt.Errorf("action error: %s", act)
		}
	} else {
		return fmt.Errorf("command error: %s", cmd)
	}

	return nil
}

func (ls *LocalStorage) Set() (err error) {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	if ls.expire > 0 {
		_, err = redisConn.Do("SET", ls.key, ls.value, "EX", ls.expire)
	} else {
		_, err = redisConn.Do("SET", ls.key, ls.value)
	}
	return
}

func (ls *LocalStorage) Get() error {
	return nil
}
func (ls *LocalStorage) Del() error {
	redisConn := Redis.Pool.Get()
	defer redisConn.Close()
	_, err := redisConn.Do("DEL", ls.key)
	return err
}
<|endoftext|>


<commit_before>package util

import (
	"github.com/ghts/ghts/lib"
	"github.com/ghts/ghts/lib/daily_price_data"
	"github.com/ghts/ghts/lib/krx_time"
	xt "github.com/ghts/ghts/xing/base"
	xing "github.com/ghts/ghts/xing/go"

	"database/sql"
	"time"
)

func F당일_일일_가격정보_수집(db *sql.DB) (에러 error) {
	if krx.F한국증시_동시호가_시간임() || krx.F한국증시_정규_거래_시간임() {
		lib.F문자열_출력("장 중에는 정확한 데이터를 수집할 수 없습니다.")
		return
	}

	daily_price_data.F일일_가격정보_테이블_생성(db)

	당일 := lib.F일자2정수(xing.F당일())
	현재가_맵, 에러 := xing.TrT8407_현물_멀티_현재가_조회_전종목()
	lib.F확인(에러)

	for 종목코드, 값 := range 현재가_맵 {
		s := new(daily_price_data.S일일_가격정보)
		s.M종목코드 = 종목코드
		s.M일자 = 당일
		s.M시가 = float64(값.M시가)
		s.M고가 = float64(값.M고가)
		s.M저가 = float64(값.M저가)
		s.M종가 = float64(값.M현재가)
		s.M거래량 = float64(값.M누적_거래량)

		종목별_일일_가격정보_모음, 에러 := daily_price_data.New종목별_일일_가격정보_모음([]*daily_price_data.S일일_가격정보{s})
		lib.F확인(에러)

		lib.F확인(종목별_일일_가격정보_모음.DB저장(db))
	}

	lib.F문자열_출력("당일 가격정보 수집 완료.")

	return nil
}

func F일개월_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	var 시작일, 종료일 time.Time
	var 종목별_일일_가격정보_모음 *daily_price_data.S종목별_일일_가격정보_모음

	daily_price_data.F일일_가격정보_테이블_생성(db)

	for i, 종목코드 := range 종목코드_모음 {
		for {
			if xing.C32_재시작_실행_중.G값() {
				lib.F대기(lib.P10초)
				continue
			}

			break
		}

		종목별_일일_가격정보_모음, 에러 = daily_price_data.New종목별_일일_가격정보_모음_DB읽기(db, 종목코드)
		lib.F확인(에러)

		// 시작일 설정. 데이터 수량이 1개이나 100개이나 소요 시간은 비슷함.
		시작일 = lib.F지금().AddDate(0, 0, -31)

		// 종료일 설정
		if lib.F지금().After(xing.F당일().Add(15*lib.P1시간 + lib.P30분)) {
			종료일 = xing.F당일()
		} else {
			종료일 = xing.F전일()
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, 종료일, 종목별_일일_가격정보_모음, i)
	}

	return nil
}

func F일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	var 시작일, 종료일, 마지막_저장일 time.Time
	var 종목별_일일_가격정보_모음 *daily_price_data.S종목별_일일_가격정보_모음

	daily_price_data.F일일_가격정보_테이블_생성(db)

	for i, 종목코드 := range 종목코드_모음 {
		for {
			if xing.C32_재시작_실행_중.G값() {
				lib.F대기(lib.P10초)
				continue
			}

			break
		}

		종목별_일일_가격정보_모음, 에러 = daily_price_data.New종목별_일일_가격정보_모음_DB읽기(db, 종목코드)
		lib.F확인(에러)

		// 시작일 설정
		시작일 = lib.F지금().AddDate(-30, 0, 0)
		if 에러 == nil && len(종목별_일일_가격정보_모음.M저장소) > 0 {
			// lib.S종목별_일일_가격정보_모음 는 일자 순서로 정렬되어 있음.
			마지막_저장일 = 종목별_일일_가격정보_모음.M저장소[len(종목별_일일_가격정보_모음.M저장소)-1].G일자2()
			시작일 = 마지막_저장일.AddDate(0, 0, 1)
		}

		if 시작일.After(xing.F당일()) {
			lib.F문자열_출력("%v [%v] : 최신 데이터 업데이트.", i, 종목코드)
			continue
		}

		// 종료일 설정
		if lib.F지금().After(xing.F당일().Add(15*lib.P1시간 + lib.P30분)) {
			종료일 = xing.F당일()
		} else {
			종료일 = xing.F전일()
		}

		if 시작일.After(종료일) {
			continue
		} else if 시작일.Equal(종료일) { // 시작일과 종료일이 같으면 수천 개의 데이터를 불러오는 현상이 있음.
			시작일 = 시작일.AddDate(0, 0, -1)
		}

		// 데이터 수량이 1개이나 100개이나 소요 시간은 비슷함.
		if 시작일.After(lib.F금일().AddDate(0, 0, -14)) {
			시작일 = lib.F금일().AddDate(0, 0, -14)
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, 종료일, 종목별_일일_가격정보_모음, i)
	}

	return nil
}

func f일일_가격정보_수집_도우미(db *sql.DB,
	종목코드 string, 시작일, 종료일 time.Time,
	종목별_일일_가격정보_모음 *daily_price_data.S종목별_일일_가격정보_모음, i int) {
	// 데이터 수집
	값_모음, 에러 := xing.TrT8413_현물_차트_일주월(종목코드, 시작일, 종료일, xt.P일주월_일)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	} else if len(값_모음) == 0 {
		lib.F체크포인트(i, 종목코드, "추가 저장할 데이터가 없음.")
		return // 추가 저장할 데이터가 없음.
	}

	일일_가격정보_슬라이스 := make([]*daily_price_data.S일일_가격정보, len(값_모음))

	for i, 일일_데이터 := range 값_모음 {
		일일_가격정보_슬라이스[i] = daily_price_data.New일일_가격정보(
			일일_데이터.M종목코드,
			일일_데이터.M일자,
			일일_데이터.M시가,
			일일_데이터.M고가,
			일일_데이터.M저가,
			일일_데이터.M종가,
			일일_데이터.M거래량)
	}

	lib.F문자열_출력("%v %v %v~%v %v개", i, 종목코드, 시작일.Format(lib.P일자_형식), 종료일.Format(lib.P일자_형식), len(값_모음))

	종목별_일일_가격정보_모음, 에러 = daily_price_data.New종목별_일일_가격정보_모음(일일_가격정보_슬라이스)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	}

	lib.F확인(종목별_일일_가격정보_모음.DB저장(db))
}
<commit_msg>불필요한 출력 문자열 비활성화<commit_after>package util

import (
	"github.com/ghts/ghts/lib"
	"github.com/ghts/ghts/lib/daily_price_data"
	"github.com/ghts/ghts/lib/krx_time"
	xt "github.com/ghts/ghts/xing/base"
	xing "github.com/ghts/ghts/xing/go"

	"database/sql"
	"time"
)

func F당일_일일_가격정보_수집(db *sql.DB) (에러 error) {
	if krx.F한국증시_동시호가_시간임() || krx.F한국증시_정규_거래_시간임() {
		lib.F문자열_출력("장 중에는 정확한 데이터를 수집할 수 없습니다.")
		return
	}

	daily_price_data.F일일_가격정보_테이블_생성(db)

	당일 := lib.F일자2정수(xing.F당일())
	현재가_맵, 에러 := xing.TrT8407_현물_멀티_현재가_조회_전종목()
	lib.F확인(에러)

	for 종목코드, 값 := range 현재가_맵 {
		s := new(daily_price_data.S일일_가격정보)
		s.M종목코드 = 종목코드
		s.M일자 = 당일
		s.M시가 = float64(값.M시가)
		s.M고가 = float64(값.M고가)
		s.M저가 = float64(값.M저가)
		s.M종가 = float64(값.M현재가)
		s.M거래량 = float64(값.M누적_거래량)

		종목별_일일_가격정보_모음, 에러 := daily_price_data.New종목별_일일_가격정보_모음([]*daily_price_data.S일일_가격정보{s})
		lib.F확인(에러)

		lib.F확인(종목별_일일_가격정보_모음.DB저장(db))
	}

	lib.F문자열_출력("당일 가격정보 수집 완료.")

	return nil
}

func F일개월_일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	var 시작일, 종료일 time.Time
	var 종목별_일일_가격정보_모음 *daily_price_data.S종목별_일일_가격정보_모음

	daily_price_data.F일일_가격정보_테이블_생성(db)

	for i, 종목코드 := range 종목코드_모음 {
		for {
			if xing.C32_재시작_실행_중.G값() {
				lib.F대기(lib.P10초)
				continue
			}

			break
		}

		종목별_일일_가격정보_모음, 에러 = daily_price_data.New종목별_일일_가격정보_모음_DB읽기(db, 종목코드)
		lib.F확인(에러)

		// 시작일 설정. 데이터 수량이 1개이나 100개이나 소요 시간은 비슷함.
		시작일 = lib.F지금().AddDate(0, 0, -31)

		// 종료일 설정
		if lib.F지금().After(xing.F당일().Add(15*lib.P1시간 + lib.P30분)) {
			종료일 = xing.F당일()
		} else {
			종료일 = xing.F전일()
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, 종료일, 종목별_일일_가격정보_모음, i)
	}

	return nil
}

func F일일_가격정보_수집(db *sql.DB, 종목코드_모음 []string) (에러 error) {
	var 시작일, 종료일, 마지막_저장일 time.Time
	var 종목별_일일_가격정보_모음 *daily_price_data.S종목별_일일_가격정보_모음

	daily_price_data.F일일_가격정보_테이블_생성(db)

	for i, 종목코드 := range 종목코드_모음 {
		for {
			if xing.C32_재시작_실행_중.G값() {
				lib.F대기(lib.P10초)
				continue
			}

			break
		}

		종목별_일일_가격정보_모음, 에러 = daily_price_data.New종목별_일일_가격정보_모음_DB읽기(db, 종목코드)
		lib.F확인(에러)

		// 시작일 설정
		시작일 = lib.F지금().AddDate(-30, 0, 0)
		if 에러 == nil && len(종목별_일일_가격정보_모음.M저장소) > 0 {
			// lib.S종목별_일일_가격정보_모음 는 일자 순서로 정렬되어 있음.
			마지막_저장일 = 종목별_일일_가격정보_모음.M저장소[len(종목별_일일_가격정보_모음.M저장소)-1].G일자2()
			시작일 = 마지막_저장일.AddDate(0, 0, 1)
		}

		if 시작일.After(xing.F당일()) {
			//lib.F문자열_출력("%v [%v] : 최신 데이터 업데이트.", i, 종목코드)
			continue
		}

		// 종료일 설정
		if lib.F지금().After(xing.F당일().Add(15*lib.P1시간 + lib.P30분)) {
			종료일 = xing.F당일()
		} else {
			종료일 = xing.F전일()
		}

		if 시작일.After(종료일) {
			continue
		} else if 시작일.Equal(종료일) { // 시작일과 종료일이 같으면 수천 개의 데이터를 불러오는 현상이 있음.
			시작일 = 시작일.AddDate(0, 0, -1)
		}

		// 데이터 수량이 1개이나 100개이나 소요 시간은 비슷함.
		if 시작일.After(lib.F금일().AddDate(0, 0, -14)) {
			시작일 = lib.F금일().AddDate(0, 0, -14)
		}

		f일일_가격정보_수집_도우미(db, 종목코드, 시작일, 종료일, 종목별_일일_가격정보_모음, i)
	}

	return nil
}

func f일일_가격정보_수집_도우미(db *sql.DB,
	종목코드 string, 시작일, 종료일 time.Time,
	종목별_일일_가격정보_모음 *daily_price_data.S종목별_일일_가격정보_모음, i int) {
	// 데이터 수집
	값_모음, 에러 := xing.TrT8413_현물_차트_일주월(종목코드, 시작일, 종료일, xt.P일주월_일)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	} else if len(값_모음) == 0 {
		lib.F체크포인트(i, 종목코드, "추가 저장할 데이터가 없음.")
		return // 추가 저장할 데이터가 없음.
	}

	일일_가격정보_슬라이스 := make([]*daily_price_data.S일일_가격정보, len(값_모음))

	for i, 일일_데이터 := range 값_모음 {
		일일_가격정보_슬라이스[i] = daily_price_data.New일일_가격정보(
			일일_데이터.M종목코드,
			일일_데이터.M일자,
			일일_데이터.M시가,
			일일_데이터.M고가,
			일일_데이터.M저가,
			일일_데이터.M종가,
			일일_데이터.M거래량)
	}

	lib.F문자열_출력("%v %v %v~%v %v개", i, 종목코드, 시작일.Format(lib.P일자_형식), 종료일.Format(lib.P일자_형식), len(값_모음))

	종목별_일일_가격정보_모음, 에러 = daily_price_data.New종목별_일일_가격정보_모음(일일_가격정보_슬라이스)
	if 에러 != nil {
		lib.F에러_출력(에러)
		return
	}

	lib.F확인(종목별_일일_가격정보_모음.DB저장(db))
}
<|endoftext|>


<commit_before>package xmmsclient

import (
	"bytes"
	"encoding/binary"
)

func serializeInt(i XmmsInt, buffer *bytes.Buffer) error {
	return binary.Write(buffer, binary.BigEndian, i)
}

func serializeString(s []byte, buffer *bytes.Buffer) error {
	err := binary.Write(buffer, binary.BigEndian, uint32(len(s)+1))
	if err != nil {
		return err
	}

	err = binary.Write(buffer, binary.BigEndian, s)
	if err != nil {
		return err
	}

	err = binary.Write(buffer, binary.BigEndian, byte(0))
	if err != nil {
		return err
	}

	return nil
}

func serializeList(l XmmsList, buffer *bytes.Buffer) error {
	err := binary.Write(buffer, binary.BigEndian, TypeNone)
	if err != nil {
		return err
	}

	err = binary.Write(buffer, binary.BigEndian, uint32(len(l)))
	if err != nil {
		return err
	}

	for _, entry := range l {
		err = serializeXmmsValue(entry, buffer)
		if err != nil {
			return err
		}
	}

	return nil
}

func serializeDict(dict XmmsDict, buffer *bytes.Buffer) error {
	err := binary.Write(buffer, binary.BigEndian, uint32(len(dict)))
	if err != nil {
		return err
	}

	for k, v := range dict {
		err = serializeString([]byte(k), buffer)
		if err != nil {
			return err
		}

		err = serializeXmmsValue(v, buffer)
		if err != nil {
			return err
		}
	}

	return nil
}

func serializeXmmsValue(value XmmsValue, buffer *bytes.Buffer) (err error) {
	switch value.(type) {
	case XmmsInt:
		err = binary.Write(buffer, binary.BigEndian, TypeInt64)
		if err == nil {
			serializeInt(value.(XmmsInt), buffer)
		}
	case XmmsString:
		err = binary.Write(buffer, binary.BigEndian, TypeString)
		if err == nil {
			serializeString([]byte(value.(XmmsString)), buffer)
		}
	case XmmsError:
		err = binary.Write(buffer, binary.BigEndian, TypeError)
		if err == nil {
			serializeString([]byte(value.(XmmsError)), buffer)
		}
	case XmmsDict:
		err = binary.Write(buffer, binary.BigEndian, TypeDict)
		if err == nil {
			serializeDict(value.(XmmsDict), buffer)
		}
	case XmmsList:
		err = binary.Write(buffer, binary.BigEndian, TypeList)
		if err == nil {
			serializeList(value.(XmmsList), buffer)
		}
	}
	return nil
}
<commit_msg>Make it possible to serialize any list.<commit_after>package xmmsclient

import (
	"bytes"
	"encoding/binary"
)

type listProducer func(buffer *bytes.Buffer) error

func serializeInt(i XmmsInt, buffer *bytes.Buffer) error {
	return binary.Write(buffer, binary.BigEndian, i)
}

func serializeString(s []byte, buffer *bytes.Buffer) error {
	err := binary.Write(buffer, binary.BigEndian, uint32(len(s)+1))
	if err != nil {
		return err
	}

	err = binary.Write(buffer, binary.BigEndian, s)
	if err != nil {
		return err
	}

	err = binary.Write(buffer, binary.BigEndian, byte(0))
	if err != nil {
		return err
	}

	return nil
}

func serializeAnyList(buffer *bytes.Buffer, length int, restrict uint32, producer listProducer) error {
	err := binary.Write(buffer, binary.BigEndian, restrict)
	if err != nil {
		return err
	}

	err = binary.Write(buffer, binary.BigEndian, uint32(length))
	if err != nil {
		return err
	}

	return producer(buffer)
}

func serializeList(list XmmsList, buffer *bytes.Buffer) error {
	return serializeAnyList(buffer, len(list), TypeNone,
		func(buffer *bytes.Buffer) error {
			for _, entry := range list {
				err := serializeXmmsValue(entry, buffer)
				if err != nil {
					return err
				}
			}
			return nil
		},
	)
}

func serializeDict(dict XmmsDict, buffer *bytes.Buffer) error {
	err := binary.Write(buffer, binary.BigEndian, uint32(len(dict)))
	if err != nil {
		return err
	}

	for k, v := range dict {
		err = serializeString([]byte(k), buffer)
		if err != nil {
			return err
		}

		err = serializeXmmsValue(v, buffer)
		if err != nil {
			return err
		}
	}

	return nil
}

func serializeXmmsValue(value XmmsValue, buffer *bytes.Buffer) (err error) {
	switch value.(type) {
	case XmmsInt:
		err = binary.Write(buffer, binary.BigEndian, TypeInt64)
		if err == nil {
			serializeInt(value.(XmmsInt), buffer)
		}
	case XmmsString:
		err = binary.Write(buffer, binary.BigEndian, TypeString)
		if err == nil {
			serializeString([]byte(value.(XmmsString)), buffer)
		}
	case XmmsError:
		err = binary.Write(buffer, binary.BigEndian, TypeError)
		if err == nil {
			serializeString([]byte(value.(XmmsError)), buffer)
		}
	case XmmsDict:
		err = binary.Write(buffer, binary.BigEndian, TypeDict)
		if err == nil {
			serializeDict(value.(XmmsDict), buffer)
		}
	case XmmsList:
		err = binary.Write(buffer, binary.BigEndian, TypeList)
		if err == nil {
			serializeList(value.(XmmsList), buffer)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package cache

import (
	"sort"
)

type SplitSizeChild struct {
	MaxSize int
	Cache   Cache
}

// A SplitSize contains multiple caches, and sends objects to different
// caches depending on the size of the item.
type SplitSize []SplitSizeChild

func NewSplitSize(children ...SplitSizeChild) SplitSize {
	s := SplitSize(children)
	sort.Sort(s)
	return s
}

func (c SplitSize) AddChildCache(maxSize int, cache Cache) SplitSize {
	c = append(c, SplitSizeChild{maxSize, cache})
	sort.Sort(c)
	return c
}

// Get an item from the cache, checking all caches.
func (c SplitSize) Get(path string, filler Filler) (o Object, err error) {
	for _, child := range c {
		o, err = child.Cache.Get(path, nil)
		if err == nil {
			return
		}
	}

	if err != nil {
		o, err = filler.Fill(c, path)
	}

	return
}

// Set adds an object to the appropriate cache for the size of the object.
// Note that it is valid for no cache to be large enough.
func (c SplitSize) Set(path string, object Object) error {
	objectSize := len(object.Data)
	for _, child := range c {
		if objectSize < child.MaxSize {
			return child.Cache.Set(path, object)
		}
	}

	// No cache is large enough to hold this object, but that's ok.
	return nil
}

func (c SplitSize) Del(path string) {
	for _, child := range c {
		child.Cache.Del(path)
	}
}

func (c SplitSize) Less(i, j int) bool {
	return c[i].MaxSize < c[j].MaxSize
}

func (c SplitSize) Len() int {
	return len(c)
}

func (c SplitSize) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}
<commit_msg>Allow MaxSize of 0 for no limit<commit_after>package cache

import (
	"sort"
)

type SplitSizeChild struct {
	MaxSize int
	Cache   Cache
}

// A SplitSize contains multiple caches, and sends objects to different
// caches depending on the size of the item.
type SplitSize []SplitSizeChild

func NewSplitSize(children ...SplitSizeChild) SplitSize {
	s := SplitSize(children)
	sort.Sort(s)
	return s
}

func (c SplitSize) AddChildCache(maxSize int, cache Cache) SplitSize {
	c = append(c, SplitSizeChild{maxSize, cache})
	sort.Sort(c)
	return c
}

// Get an item from the cache, checking all caches.
func (c SplitSize) Get(path string, filler Filler) (o Object, err error) {
	for _, child := range c {
		o, err = child.Cache.Get(path, nil)
		if err == nil {
			return
		}
	}

	if err != nil {
		o, err = filler.Fill(c, path)
	}

	return
}

// Set adds an object to the appropriate cache for the size of the object.
// Note that it is valid for no cache to be large enough.
func (c SplitSize) Set(path string, object Object) error {
	objectSize := len(object.Data)
	for _, child := range c {
		if objectSize < child.MaxSize || child.MaxSize == 0 {
			return child.Cache.Set(path, object)
		}
	}

	// No cache is large enough to hold this object, but that's ok.
	return nil
}

func (c SplitSize) Del(path string) {
	for _, child := range c {
		child.Cache.Del(path)
	}
}

func (c SplitSize) Less(i, j int) bool {
	return c[i].MaxSize < c[j].MaxSize || c[j].MaxSize == 0
}

func (c SplitSize) Len() int {
	return len(c)
}

func (c SplitSize) Swap(i, j int) {
	c[i], c[j] = c[j], c[i]
}
<|endoftext|>


<commit_before>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package rtt

import (
	"appengine"
	"appengine/datastore"
	// "appengine/urlfetch"
	// "code.google.com/p/golog2bq/log2bq"
	"code.google.com/p/google-api-go-client/bigquery/v2"
	"code.google.com/p/mlab-ns2/gae/ns/data"
	"fmt"
	"net/http"
	"time"
)

const (
	MaxDSReadPerQuery         = 1000
	MaxDSWritePerQuery        = 500
	MaxBQResponseRows         = 50000 //Response size must be less than 32MB. 100k rows occasionally caused problems.
	BigQueryBillableProjectID = "mlab-ns2"
)

// bqQueryFormat is the query used to pull RTT data from the M-Lab BigQuery
// dataset.
// NOTE: It must be formatted with Table Name, Year, Month, Day, Year, Month,
// Day to specify which day the query is being performed for.
//
// The following columns are selected:
// - Logged Time (log_time)
// - M-Lab Server IP (connection_spec.server_ip)
// - Destination IP for traceroute hop, towards client (paris_traceroute_hop.dest_ip)
// - Average of RTT in same traceroute and hop
//
// The Query is performed for entries logged on specified days and for cases
// where the field paris_traceroute_hop.rtt is not null. (RTT data exists)
// The query also excludes RTT to the client due to the variability of the last
// hop.
//
// The result is grouped by time and the IPs such that multiple traceroute rtt
// entries can be averaged.

// The result is ordered by server and client IPs to allow for more efficient
// traversal of response entries.
const bqQueryFormat = `SELECT
		log_time,
		connection_spec.server_ip,
		paris_traceroute_hop.dest_ip,
		AVG(paris_traceroute_hop.rtt) AS rtt
	FROM [%s]
	WHERE
		project = 3 AND
		log_time > %d AND
		log_time < %d AND
		log_time IS NOT NULL AND
		connection_spec.server_ip IS NOT NULL AND
		paris_traceroute_hop.dest_ip IS NOT NULL AND
		paris_traceroute_hop.rtt IS NOT NULL AND
		connection_spec.client_ip != paris_traceroute_hop.dest_ip
	GROUP EACH BY
		log_time,
		connection_spec.server_ip,
		paris_traceroute_hop.dest_ip;`

// bqInit authenticates a transport using OAuth and returns a *bigquery.Service
// with which to make queries to bigquery.
func bqInit(r *http.Request) (*bigquery.Service, error) {
	c := appengine.NewContext(r)

	// Get transport from log2bq's utility function GAETransport
	transport, err := log2bq.GAETransport(c, bigquery.BigqueryScope)
	if err != nil {
		return nil, err
	}

	// Set maximum urlfetch request deadline
	transport.Transport = &urlfetch.Transport{
		Context:  c,
		Deadline: 10 * time.Minute,
	}

	client, err := transport.Client()
	if err != nil {
		return nil, err
	}

	service, err := bigquery.New(client)
	return service, err
}

// BQImportDay queries BigQuery for RTT data from a specific day and stores new
// data into datastore
func BQImportDay(r *http.Request, t time.Time) {
	c := appengine.NewContext(r)
	service, err := bqInit(r)
	if err != nil {
		c.Errorf("rtt: BQImportDay.bqInit: %s", err)
		return
	}

	// Format strings to insert into bqQueryFormat
	tableName := fmt.Sprintf("measurement-lab:m_lab.%.4d_%.2d", t.Year(), t.Month())
	dateStr := t.Format(dateFormat)
	startTime, _ := time.Parse(timeFormat, dateStr+" 00:00:00")
	endTime, _ := time.Parse(timeFormat, dateStr+" 23:59:59")

	// Construct query
	qText := fmt.Sprintf(bqQueryFormat, tableName, startTime.Unix(), endTime.Unix())
	q := &bigquery.QueryRequest{
		Query:         qText,
		MaxResults:    MaxBQResponseRows,
		TimeoutMs:     600000,
		UseQueryCache: true,
	}
	c.Debugf("rtt: BQImportDay.qText (%s): %s", dateStr, qText)

	// Make first query to BigQuery
	jobsService := bigquery.NewJobsService(service)
	queryCall := jobsService.Query(BigQueryBillableProjectID, q)
	response, err := queryCall.Do()
	if err != nil {
		c.Errorf("rtt: BQImportDay.bigquery.JobsService.Query: %s", err)
		return
	}
	c.Infof("rtt: Received %d rows in query response (Total: %d rows).", len(response.Rows), response.TotalRows)

	newCGs := make(map[string]*ClientGroup)
	sliverTools, err := data.GetSliverTools(c)
	if err != nil {
		c.Errorf("rtt: BQImportDay.data.GetSliverTools: %s", err)
	}
	sliverIPMap := makeMapIPStrToSiteID(sliverTools)
	sliverTools = nil
	bqProcessQuery(response.Rows, sliverIPMap, newCGs)

	// Cache details from response to use in subsequent requests if any.
	projID := response.JobReference.ProjectId
	jobID := response.JobReference.JobId
	pageToken := response.PageToken
	n := len(response.Rows)
	totalN := int(response.TotalRows)
	response = nil

	// Request for more results if not all results returned.
	if n < totalN {
		// Make further requests
		getQueryResultsCall := jobsService.GetQueryResults(projID, jobID)
		var respMore *bigquery.GetQueryResultsResponse

		for n < totalN { // Make requests until total number of rows queried.
			getQueryResultsCall.MaxResults(MaxBQResponseRows)
			getQueryResultsCall.PageToken(pageToken)

			respMore, err = getQueryResultsCall.Do()
			if err != nil {
				c.Errorf("rtt: BQImportDay.bigquery.JobsGetQueryResponseCall: %s", err)
				return
			}
			pageToken = respMore.PageToken // Update pageToken to get next page.

			n += len(respMore.Rows)
			c.Infof("rtt: Received %d additional rows. (Total: %d rows)", len(respMore.Rows), n)

			bqProcessQuery(respMore.Rows, sliverIPMap, newCGs)
			respMore = nil
		}
	}

	c.Infof("rtt: Reduced %d rows to %d rows. Merging into datastore.", totalN, len(newCGs))

	bqMergeWithDatastore(c, newCGs)
}

// bqProcessQuery processes the output of the BigQuery query performed in
// BQImport and parses the response into data structures.
func bqProcessQuery(resp []*bigquery.TableRow, sliverIPMap map[string]string, newCGs map[string]*ClientGroup) {
	rows := simplifyBQResponse(resp)
	bqMergeIntoClientGroups(rows, sliverIPMap, newCGs)
}

// dsWriteChunk is a structure with which new ClientGroup lists can be split
// into lengths <= MaxDSWritePerQuery such that datastore.PutMulti works.
type dsReadChunk struct {
	Keys []*datastore.Key
	CGs  []*ClientGroup
}

// bqMergeWithDatastore takes a list of ClientGroup generated by bqProcessQuery
// and merges the new data with existing data in datastore
// TODO(gavaletz): Evaluate whether this func could be split into smaller funcs,
// also consider whether it's fine to have 3 local funcs.
func bqMergeWithDatastore(c appengine.Context, newCGs map[string]*ClientGroup) {
	// Divide GetMulti and PutMulti operations into MaxDSReadPerQuery sized
	// operations to adhere with GAE limits.
	chunks := make([]*dsReadChunk, 0)
	var thisChunk *dsReadChunk
	// newChunk creates a new dsReadChunk as necessary
	newChunk := func() {
		thisChunk = &dsReadChunk{
			Keys: make([]*datastore.Key, 0, MaxDSReadPerQuery),
			CGs:  make([]*ClientGroup, 0, MaxDSReadPerQuery),
		}
		chunks = append(chunks, thisChunk)
	}
	newChunk() // Create initial dsReadChunk

	rttKey := datastore.NewKey(c, "string", "rtt", 0, nil) // Parent key for ClientGroup entities
	for cgStr, cg := range newCGs {
		// Add into chunk
		thisChunk.Keys = append(thisChunk.Keys, datastore.NewKey(c, "ClientGroup", cgStr, 0, rttKey))
		thisChunk.CGs = append(thisChunk.CGs, cg)

		// Make sure read chunks are only as large as MaxDSReadPerQuery.
		// Create new chunk if size reached.
		if len(thisChunk.CGs) == MaxDSReadPerQuery {
			newChunk()
		}
	}

	var oldCGs []ClientGroup
	var merr appengine.MultiError
	var err error

	keysToPut := make([]*datastore.Key, 0, MaxDSWritePerQuery)
	cgsToPut := make([]ClientGroup, 0, MaxDSWritePerQuery)
	totalPutN := 0

	// processPutQueue processes a queue of newly updated ClientGroups. This is
	// done so that MaxDSWritePerQuery no. of Puts can be done to reduce the
	// number of queries to datastore and therefore the time taken to Put all
	// changes to datastore.
	processPutQueue := func() {
		totalPutN += len(cgsToPut)
		c.Infof("rtt: Putting %v records into datastore. (Total: %d rows)", len(cgsToPut), totalPutN)

		_, err = datastore.PutMulti(c, keysToPut, cgsToPut)
		if err != nil {
			c.Errorf("rtt: bqMergeWithDatastore.datastore.PutMulti: %s", err)
		}
		keysToPut = make([]*datastore.Key, 0, MaxDSWritePerQuery)
		cgsToPut = make([]ClientGroup, 0, MaxDSWritePerQuery)
	}

	// processPutQueue places a newly updated ClientGroup in a queue. This queue
	// is later processed by processPutQueue.
	queueToPut := func(idx int, keys []*datastore.Key) {
		keysToPut = append(keysToPut, keys[idx])
		cgsToPut = append(cgsToPut, oldCGs[idx])

		// Write in MaxDSWritePerQuery chunks to adhere with GAE limits.
		if len(keysToPut) == MaxDSWritePerQuery {
			processPutQueue()
		}
	}

	// Process chunk by chunk
	for _, chunk := range chunks {
		oldCGs = make([]ClientGroup, len(chunk.CGs))
		err = datastore.GetMulti(c, chunk.Keys, oldCGs) // Get existing ClientGroup data

		switch err.(type) {
		case appengine.MultiError: // Multiple errors, deal with individually
			merr = err.(appengine.MultiError)
			for i, e := range merr { // Range over errors
				switch e {
				case datastore.ErrNoSuchEntity: // New entry
					oldCGs[i] = *chunk.CGs[i]
					queueToPut(i, chunk.Keys)
				case nil: // Entry exists, merge new data with old data
					// If for some reason data is corrupted and nil is returned
					if oldCGs[i].SiteRTTs == nil {
						oldCGs[i] = *chunk.CGs[i]
						queueToPut(i, chunk.Keys)
					} else {
						changed, err := MergeClientGroups(&oldCGs[i], chunk.CGs[i])
						if err != nil {
							c.Errorf("rtt: bqMergeWithDatastore.MergeClientGroups: %s", err)
						}
						if changed {
							queueToPut(i, chunk.Keys)
						}
					}
				default: // Other unknown error
					c.Errorf("rtt: bqMergeWithDatastore.datastore.GetMulti: %s", err)
				}
			}
		case nil: // No errors, data exists so merge with old data
			for i, _ := range oldCGs {
				changed, err := MergeClientGroups(&oldCGs[i], chunk.CGs[i])
				if err != nil {
					c.Errorf("rtt: bqMergeWithDatastore.MergeClientGroups: %s", err)
				}
				if changed {
					queueToPut(i, chunk.Keys)
				}
			}
		default: // Other unknown errors from GetMulti
			c.Errorf("rtt: bqMergeWithDatastore.datastore.GetMulti: %s", err)
		}
	}

	// Process remaining Put operations.
	if len(keysToPut) > 0 {
		processPutQueue()
	}

	c.Infof("rtt: Completed merging in %d rows from BigQuery.", len(newCGs))
}
<commit_msg>rtt: Enable imports to fix e4006b744b<commit_after>// Copyright 2013 M-Lab
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package rtt

import (
	"appengine"
	"appengine/datastore"
	"appengine/urlfetch"
	"code.google.com/p/golog2bq/log2bq"
	"code.google.com/p/google-api-go-client/bigquery/v2"
	"code.google.com/p/mlab-ns2/gae/ns/data"
	"fmt"
	"net/http"
	"time"
)

const (
	MaxDSReadPerQuery         = 1000
	MaxDSWritePerQuery        = 500
	MaxBQResponseRows         = 50000 //Response size must be less than 32MB. 100k rows occasionally caused problems.
	BigQueryBillableProjectID = "mlab-ns2"
)

// bqQueryFormat is the query used to pull RTT data from the M-Lab BigQuery
// dataset.
// NOTE: It must be formatted with Table Name, Year, Month, Day, Year, Month,
// Day to specify which day the query is being performed for.
//
// The following columns are selected:
// - Logged Time (log_time)
// - M-Lab Server IP (connection_spec.server_ip)
// - Destination IP for traceroute hop, towards client (paris_traceroute_hop.dest_ip)
// - Average of RTT in same traceroute and hop
//
// The Query is performed for entries logged on specified days and for cases
// where the field paris_traceroute_hop.rtt is not null. (RTT data exists)
// The query also excludes RTT to the client due to the variability of the last
// hop.
//
// The result is grouped by time and the IPs such that multiple traceroute rtt
// entries can be averaged.

// The result is ordered by server and client IPs to allow for more efficient
// traversal of response entries.
const bqQueryFormat = `SELECT
		log_time,
		connection_spec.server_ip,
		paris_traceroute_hop.dest_ip,
		AVG(paris_traceroute_hop.rtt) AS rtt
	FROM [%s]
	WHERE
		project = 3 AND
		log_time > %d AND
		log_time < %d AND
		log_time IS NOT NULL AND
		connection_spec.server_ip IS NOT NULL AND
		paris_traceroute_hop.dest_ip IS NOT NULL AND
		paris_traceroute_hop.rtt IS NOT NULL AND
		connection_spec.client_ip != paris_traceroute_hop.dest_ip
	GROUP EACH BY
		log_time,
		connection_spec.server_ip,
		paris_traceroute_hop.dest_ip;`

// bqInit authenticates a transport using OAuth and returns a *bigquery.Service
// with which to make queries to bigquery.
func bqInit(r *http.Request) (*bigquery.Service, error) {
	c := appengine.NewContext(r)

	// Get transport from log2bq's utility function GAETransport
	transport, err := log2bq.GAETransport(c, bigquery.BigqueryScope)
	if err != nil {
		return nil, err
	}

	// Set maximum urlfetch request deadline
	transport.Transport = &urlfetch.Transport{
		Context:  c,
		Deadline: 10 * time.Minute,
	}

	client, err := transport.Client()
	if err != nil {
		return nil, err
	}

	service, err := bigquery.New(client)
	return service, err
}

// BQImportDay queries BigQuery for RTT data from a specific day and stores new
// data into datastore
func BQImportDay(r *http.Request, t time.Time) {
	c := appengine.NewContext(r)
	service, err := bqInit(r)
	if err != nil {
		c.Errorf("rtt: BQImportDay.bqInit: %s", err)
		return
	}

	// Format strings to insert into bqQueryFormat
	tableName := fmt.Sprintf("measurement-lab:m_lab.%.4d_%.2d", t.Year(), t.Month())
	dateStr := t.Format(dateFormat)
	startTime, _ := time.Parse(timeFormat, dateStr+" 00:00:00")
	endTime, _ := time.Parse(timeFormat, dateStr+" 23:59:59")

	// Construct query
	qText := fmt.Sprintf(bqQueryFormat, tableName, startTime.Unix(), endTime.Unix())
	q := &bigquery.QueryRequest{
		Query:         qText,
		MaxResults:    MaxBQResponseRows,
		TimeoutMs:     600000,
		UseQueryCache: true,
	}
	c.Debugf("rtt: BQImportDay.qText (%s): %s", dateStr, qText)

	// Make first query to BigQuery
	jobsService := bigquery.NewJobsService(service)
	queryCall := jobsService.Query(BigQueryBillableProjectID, q)
	response, err := queryCall.Do()
	if err != nil {
		c.Errorf("rtt: BQImportDay.bigquery.JobsService.Query: %s", err)
		return
	}
	c.Infof("rtt: Received %d rows in query response (Total: %d rows).", len(response.Rows), response.TotalRows)

	newCGs := make(map[string]*ClientGroup)
	sliverTools, err := data.GetSliverTools(c)
	if err != nil {
		c.Errorf("rtt: BQImportDay.data.GetSliverTools: %s", err)
	}
	sliverIPMap := makeMapIPStrToSiteID(sliverTools)
	sliverTools = nil
	bqProcessQuery(response.Rows, sliverIPMap, newCGs)

	// Cache details from response to use in subsequent requests if any.
	projID := response.JobReference.ProjectId
	jobID := response.JobReference.JobId
	pageToken := response.PageToken
	n := len(response.Rows)
	totalN := int(response.TotalRows)
	response = nil

	// Request for more results if not all results returned.
	if n < totalN {
		// Make further requests
		getQueryResultsCall := jobsService.GetQueryResults(projID, jobID)
		var respMore *bigquery.GetQueryResultsResponse

		for n < totalN { // Make requests until total number of rows queried.
			getQueryResultsCall.MaxResults(MaxBQResponseRows)
			getQueryResultsCall.PageToken(pageToken)

			respMore, err = getQueryResultsCall.Do()
			if err != nil {
				c.Errorf("rtt: BQImportDay.bigquery.JobsGetQueryResponseCall: %s", err)
				return
			}
			pageToken = respMore.PageToken // Update pageToken to get next page.

			n += len(respMore.Rows)
			c.Infof("rtt: Received %d additional rows. (Total: %d rows)", len(respMore.Rows), n)

			bqProcessQuery(respMore.Rows, sliverIPMap, newCGs)
			respMore = nil
		}
	}

	c.Infof("rtt: Reduced %d rows to %d rows. Merging into datastore.", totalN, len(newCGs))

	bqMergeWithDatastore(c, newCGs)
}

// bqProcessQuery processes the output of the BigQuery query performed in
// BQImport and parses the response into data structures.
func bqProcessQuery(resp []*bigquery.TableRow, sliverIPMap map[string]string, newCGs map[string]*ClientGroup) {
	rows := simplifyBQResponse(resp)
	bqMergeIntoClientGroups(rows, sliverIPMap, newCGs)
}

// dsWriteChunk is a structure with which new ClientGroup lists can be split
// into lengths <= MaxDSWritePerQuery such that datastore.PutMulti works.
type dsReadChunk struct {
	Keys []*datastore.Key
	CGs  []*ClientGroup
}

// bqMergeWithDatastore takes a list of ClientGroup generated by bqProcessQuery
// and merges the new data with existing data in datastore
// TODO(gavaletz): Evaluate whether this func could be split into smaller funcs,
// also consider whether it's fine to have 3 local funcs.
func bqMergeWithDatastore(c appengine.Context, newCGs map[string]*ClientGroup) {
	// Divide GetMulti and PutMulti operations into MaxDSReadPerQuery sized
	// operations to adhere with GAE limits.
	chunks := make([]*dsReadChunk, 0)
	var thisChunk *dsReadChunk
	// newChunk creates a new dsReadChunk as necessary
	newChunk := func() {
		thisChunk = &dsReadChunk{
			Keys: make([]*datastore.Key, 0, MaxDSReadPerQuery),
			CGs:  make([]*ClientGroup, 0, MaxDSReadPerQuery),
		}
		chunks = append(chunks, thisChunk)
	}
	newChunk() // Create initial dsReadChunk

	rttKey := datastore.NewKey(c, "string", "rtt", 0, nil) // Parent key for ClientGroup entities
	for cgStr, cg := range newCGs {
		// Add into chunk
		thisChunk.Keys = append(thisChunk.Keys, datastore.NewKey(c, "ClientGroup", cgStr, 0, rttKey))
		thisChunk.CGs = append(thisChunk.CGs, cg)

		// Make sure read chunks are only as large as MaxDSReadPerQuery.
		// Create new chunk if size reached.
		if len(thisChunk.CGs) == MaxDSReadPerQuery {
			newChunk()
		}
	}

	var oldCGs []ClientGroup
	var merr appengine.MultiError
	var err error

	keysToPut := make([]*datastore.Key, 0, MaxDSWritePerQuery)
	cgsToPut := make([]ClientGroup, 0, MaxDSWritePerQuery)
	totalPutN := 0

	// processPutQueue processes a queue of newly updated ClientGroups. This is
	// done so that MaxDSWritePerQuery no. of Puts can be done to reduce the
	// number of queries to datastore and therefore the time taken to Put all
	// changes to datastore.
	processPutQueue := func() {
		totalPutN += len(cgsToPut)
		c.Infof("rtt: Putting %v records into datastore. (Total: %d rows)", len(cgsToPut), totalPutN)

		_, err = datastore.PutMulti(c, keysToPut, cgsToPut)
		if err != nil {
			c.Errorf("rtt: bqMergeWithDatastore.datastore.PutMulti: %s", err)
		}
		keysToPut = make([]*datastore.Key, 0, MaxDSWritePerQuery)
		cgsToPut = make([]ClientGroup, 0, MaxDSWritePerQuery)
	}

	// processPutQueue places a newly updated ClientGroup in a queue. This queue
	// is later processed by processPutQueue.
	queueToPut := func(idx int, keys []*datastore.Key) {
		keysToPut = append(keysToPut, keys[idx])
		cgsToPut = append(cgsToPut, oldCGs[idx])

		// Write in MaxDSWritePerQuery chunks to adhere with GAE limits.
		if len(keysToPut) == MaxDSWritePerQuery {
			processPutQueue()
		}
	}

	// Process chunk by chunk
	for _, chunk := range chunks {
		oldCGs = make([]ClientGroup, len(chunk.CGs))
		err = datastore.GetMulti(c, chunk.Keys, oldCGs) // Get existing ClientGroup data

		switch err.(type) {
		case appengine.MultiError: // Multiple errors, deal with individually
			merr = err.(appengine.MultiError)
			for i, e := range merr { // Range over errors
				switch e {
				case datastore.ErrNoSuchEntity: // New entry
					oldCGs[i] = *chunk.CGs[i]
					queueToPut(i, chunk.Keys)
				case nil: // Entry exists, merge new data with old data
					// If for some reason data is corrupted and nil is returned
					if oldCGs[i].SiteRTTs == nil {
						oldCGs[i] = *chunk.CGs[i]
						queueToPut(i, chunk.Keys)
					} else {
						changed, err := MergeClientGroups(&oldCGs[i], chunk.CGs[i])
						if err != nil {
							c.Errorf("rtt: bqMergeWithDatastore.MergeClientGroups: %s", err)
						}
						if changed {
							queueToPut(i, chunk.Keys)
						}
					}
				default: // Other unknown error
					c.Errorf("rtt: bqMergeWithDatastore.datastore.GetMulti: %s", err)
				}
			}
		case nil: // No errors, data exists so merge with old data
			for i, _ := range oldCGs {
				changed, err := MergeClientGroups(&oldCGs[i], chunk.CGs[i])
				if err != nil {
					c.Errorf("rtt: bqMergeWithDatastore.MergeClientGroups: %s", err)
				}
				if changed {
					queueToPut(i, chunk.Keys)
				}
			}
		default: // Other unknown errors from GetMulti
			c.Errorf("rtt: bqMergeWithDatastore.datastore.GetMulti: %s", err)
		}
	}

	// Process remaining Put operations.
	if len(keysToPut) > 0 {
		processPutQueue()
	}

	c.Infof("rtt: Completed merging in %d rows from BigQuery.", len(newCGs))
}
<|endoftext|>


<commit_before>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package web

import (
	"github.com/zenoss/glog"
	"github.com/zenoss/go-json-rest"
	"github.com/zenoss/serviced/domain"
	"github.com/zenoss/serviced/domain/host"
	"github.com/zenoss/serviced/rpc/agent"

	"net/url"
	"strings"
)

//restGetHosts gets all hosts. Response is map[host-id]host.Host
func restGetHosts(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	response := make(map[string]*host.Host)
	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w)
		return
	}

	hosts, err := client.GetHosts()
	if err != nil {
		glog.Errorf("Could not get hosts: %v", err)
		restServerError(w)
		return
	}
	glog.V(2).Infof("Returning %d hosts", len(hosts))
	for _, host := range hosts {
		response[host.ID] = host
		err = buildHostMonitoringProfile(host)
		if err != nil {
			restServerError(w)
			return
		}
	}

	w.WriteJson(&response)
}

//restGetHost retrieves a host. Response is Host
func restGetHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	hostID, err := url.QueryUnescape(r.PathParam("hostId"))
	if err != nil {
		restBadRequest(w)
		return
	}

	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w)
		return
	}

	host, err := client.GetHost(hostID)
	if err != nil {
		glog.Error("Could not get host: ", err)
		restServerError(w)
		return
	}

	err = buildHostMonitoringProfile(host)
	if err != nil {
		restServerError(w)
		return
	}

	glog.V(4).Infof("restGetHost: id %s, host %#v", hostID, host)
	w.WriteJson(&host)
}

//restAddHost adds a Host. Request input is host.Host
func restAddHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	var payload host.Host
	err := r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Infof("Could not decode host payload: %v", err)
		restBadRequest(w)
		return
	}
	// Save the pool ID and IP address for later. GetInfo wipes these
	ipAddr := payload.IPAddr
	parts := strings.Split(ipAddr, ":")
	hostIP := parts[0]

	agentClient, err := agent.NewClient(payload.IPAddr)
	//	remoteClient, err := serviced.NewAgentClient(payload.IPAddr)
	if err != nil {
		glog.Errorf("Could not create connection to host %s: %v", payload.IPAddr, err)
		restServerError(w)
		return
	}

	IPs := []string{}
	for _, ip := range payload.IPs {
		IPs = append(IPs, ip.IPAddress)
	}
	buildRequest := agent.BuildHostRequest{
		IP:     hostIP,
		PoolID: payload.PoolID,
	}
	host, err := agentClient.BuildHost(buildRequest)
	if err != nil {
		glog.Errorf("Unable to get remote host info: %v", err)
		restBadRequest(w)
		return
	}
	masterClient, err := ctx.getMasterClient()
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w)
		return
	}
	err = masterClient.AddHost(*host)
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w)
		return
	}
	glog.V(0).Info("Added host ", host.ID)
	w.WriteJson(&simpleResponse{"Added host", hostLinks(host.ID)})
}

//restUpdateHost updates a host. Request input is host.Host
func restUpdateHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	hostID, err := url.QueryUnescape(r.PathParam("hostId"))
	if err != nil {
		restBadRequest(w)
		return
	}
	glog.V(3).Infof("Received update request for %s", hostID)
	var payload host.Host
	err = r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Infof("Could not decode host payload: %v", err)
		restBadRequest(w)
		return
	}

	masterClient, err := ctx.getMasterClient()
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w)
		return
	}
	err = masterClient.UpdateHost(payload)
	if err != nil {
		glog.Errorf("Unable to update host: %v", err)
		restServerError(w)
		return
	}
	glog.V(1).Info("Updated host ", hostID)
	w.WriteJson(&simpleResponse{"Updated host", hostLinks(hostID)})
}

//restRemoveHost removes a host using host-id
func restRemoveHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	hostID, err := url.QueryUnescape(r.PathParam("hostId"))
	if err != nil {
		restBadRequest(w)
		return
	}

	masterClient, err := ctx.getMasterClient()
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w)
		return
	}
	err = masterClient.RemoveHost(hostID)
	if err != nil {
		glog.Errorf("Could not remove host: %v", err)
		restServerError(w)
		return
	}
	glog.V(0).Info("Removed host ", hostID)
	w.WriteJson(&simpleResponse{"Removed host", hostsLinks()})
}

type metric struct {
	ID          string
	Name        string
	Description string
}

var (
	metrics = []metric{
		metric{
			"CpuacctStat.system",
			"CPU System",
			"System CPU Usage",
		},
		metric{
			"CpuacctStat.user",
			"CPU User",
			"User CPU Usage",
		},
		metric{
			"MemoryStat.pgfault",
			"Memory Page Fault",
			"Page Fault Stats",
		},
		metric{
			"MemoryStat.rss",
			"Resident Memory",
			"Resident Memory Usage",
		},
	}
)

func buildHostMonitoringProfile(host *host.Host) error {
	host.MonitoringProfile = domain.MonitorProfile{
		Metrics: make([]domain.MetricConfig, len(metrics)),
	}

	build, err := domain.NewMetricConfigBuilder("/metrics/api/performance/query", "POST")
	if err != nil {
		glog.Errorf("Failed to create metric builder: %s", err)
		return err
	}

	for i := range metrics {
		m := &metrics[i]
		build.Metric(m.ID, m.Name).SetTag("controlplane_host_id", host.ID)
		config, err := build.Config(m.ID, m.Name, m.Description, "1h-ago")
		if err != nil {
			glog.Errorf("Failed to build metric: %s", err)
		}
		host.MonitoringProfile.Metrics[i] = *config
	}

	return nil
}
<commit_msg>Response to code review<commit_after>// Copyright 2014, The Serviced Authors. All rights reserved.
// Use of this source code is governed by a
// license that can be found in the LICENSE file.

package web

import (
	"github.com/zenoss/glog"
	"github.com/zenoss/go-json-rest"
	"github.com/zenoss/serviced/domain"
	"github.com/zenoss/serviced/domain/host"
	"github.com/zenoss/serviced/rpc/agent"

	"net/url"
	"strings"
)

//restGetHosts gets all hosts. Response is map[host-id]host.Host
func restGetHosts(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	response := make(map[string]*host.Host)
	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w)
		return
	}

	hosts, err := client.GetHosts()
	if err != nil {
		glog.Errorf("Could not get hosts: %v", err)
		restServerError(w)
		return
	}
	glog.V(2).Infof("Returning %d hosts", len(hosts))
	for _, host := range hosts {
		response[host.ID] = host
		if err := buildHostMonitoringProfile(host); err != nil {
			restServerError(w)
			return
		}
	}

	w.WriteJson(&response)
}

//restGetHost retrieves a host. Response is Host
func restGetHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	hostID, err := url.QueryUnescape(r.PathParam("hostId"))
	if err != nil {
		restBadRequest(w)
		return
	}

	client, err := ctx.getMasterClient()
	if err != nil {
		restServerError(w)
		return
	}

	host, err := client.GetHost(hostID)
	if err != nil {
		glog.Error("Could not get host: ", err)
		restServerError(w)
		return
	}

	if err := buildHostMonitoringProfile(host); err != nil {
		restServerError(w)
		return
	}

	glog.V(4).Infof("restGetHost: id %s, host %#v", hostID, host)
	w.WriteJson(&host)
}

//restAddHost adds a Host. Request input is host.Host
func restAddHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	var payload host.Host
	err := r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Infof("Could not decode host payload: %v", err)
		restBadRequest(w)
		return
	}
	// Save the pool ID and IP address for later. GetInfo wipes these
	ipAddr := payload.IPAddr
	parts := strings.Split(ipAddr, ":")
	hostIP := parts[0]

	agentClient, err := agent.NewClient(payload.IPAddr)
	//	remoteClient, err := serviced.NewAgentClient(payload.IPAddr)
	if err != nil {
		glog.Errorf("Could not create connection to host %s: %v", payload.IPAddr, err)
		restServerError(w)
		return
	}

	IPs := []string{}
	for _, ip := range payload.IPs {
		IPs = append(IPs, ip.IPAddress)
	}
	buildRequest := agent.BuildHostRequest{
		IP:     hostIP,
		PoolID: payload.PoolID,
	}
	host, err := agentClient.BuildHost(buildRequest)
	if err != nil {
		glog.Errorf("Unable to get remote host info: %v", err)
		restBadRequest(w)
		return
	}
	masterClient, err := ctx.getMasterClient()
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w)
		return
	}
	err = masterClient.AddHost(*host)
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w)
		return
	}
	glog.V(0).Info("Added host ", host.ID)
	w.WriteJson(&simpleResponse{"Added host", hostLinks(host.ID)})
}

//restUpdateHost updates a host. Request input is host.Host
func restUpdateHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	hostID, err := url.QueryUnescape(r.PathParam("hostId"))
	if err != nil {
		restBadRequest(w)
		return
	}
	glog.V(3).Infof("Received update request for %s", hostID)
	var payload host.Host
	err = r.DecodeJsonPayload(&payload)
	if err != nil {
		glog.V(1).Infof("Could not decode host payload: %v", err)
		restBadRequest(w)
		return
	}

	masterClient, err := ctx.getMasterClient()
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w)
		return
	}
	err = masterClient.UpdateHost(payload)
	if err != nil {
		glog.Errorf("Unable to update host: %v", err)
		restServerError(w)
		return
	}
	glog.V(1).Info("Updated host ", hostID)
	w.WriteJson(&simpleResponse{"Updated host", hostLinks(hostID)})
}

//restRemoveHost removes a host using host-id
func restRemoveHost(w *rest.ResponseWriter, r *rest.Request, ctx *requestContext) {
	hostID, err := url.QueryUnescape(r.PathParam("hostId"))
	if err != nil {
		restBadRequest(w)
		return
	}

	masterClient, err := ctx.getMasterClient()
	if err != nil {
		glog.Errorf("Unable to add host: %v", err)
		restServerError(w)
		return
	}
	err = masterClient.RemoveHost(hostID)
	if err != nil {
		glog.Errorf("Could not remove host: %v", err)
		restServerError(w)
		return
	}
	glog.V(0).Info("Removed host ", hostID)
	w.WriteJson(&simpleResponse{"Removed host", hostsLinks()})
}

type metric struct {
	ID          string
	Name        string
	Description string
}

var (
	metrics = []metric{
		metric{
			"CpuacctStat.system",
			"CPU System",
			"System CPU Usage",
		},
		metric{
			"CpuacctStat.user",
			"CPU User",
			"User CPU Usage",
		},
		metric{
			"MemoryStat.pgfault",
			"Memory Page Fault",
			"Page Fault Stats",
		},
		metric{
			"MemoryStat.rss",
			"Resident Memory",
			"Resident Memory Usage",
		},
	}
)

func buildHostMonitoringProfile(host *host.Host) error {
	host.MonitoringProfile = domain.MonitorProfile{
		Metrics: make([]domain.MetricConfig, len(metrics)),
	}

	build, err := domain.NewMetricConfigBuilder("/metrics/api/performance/query", "POST")
	if err != nil {
		glog.Errorf("Failed to create metric builder: %s", err)
		return err
	}

	for i := range metrics {
		build.Metric(metrics[i].ID, metrics[i].Name).SetTag("controlplane_host_id", host.ID)
		config, err := build.Config(metrics[i].ID, metrics[i].Name, metrics[i].Description, "1h-ago")
		if err != nil {
			glog.Errorf("Failed to build metric: %s", err)
			host.MonitoringProfile = domain.MonitorProfile{}
			return err
		}
		host.MonitoringProfile.Metrics[i] = *config
	}

	return nil
}
<|endoftext|>


<commit_before>/*
** Copyright [2013-2016] [Megam Systems]
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
 */
package carton

import (
	"fmt"
	ldb "github.com/megamsys/libgo/db"
	"github.com/megamsys/libgo/pairs"
	"github.com/megamsys/libgo/utils"
	"github.com/megamsys/vertice/carton/bind"
	"github.com/megamsys/vertice/meta"
	"github.com/megamsys/vertice/provision"
	"github.com/megamsys/vertice/repository"
	"gopkg.in/yaml.v2"
	"strings"
	"time"
)

const (
	DOMAIN        = "domain"
	PUBLICIPV4    = "publicipv4"
	PRIVATEIPV4   = "privateipv4"
	COMPBUCKET    = "components"
	IMAGE_VERSION = "version"
	ONECLICK      = "oneclick"
	HOSTIP        = "hostip"
  TRUE          = "true"
)

type Artifacts struct {
	Type         string          `json:"artifact_type" cql:"type"`
	Content      string          `json:"content" cql:"content"`
	Requirements pairs.JsonPairs `json:"requirements" cql:"requirements"`
}

/* Repository represents a repository managed by the manager. */
type Repo struct {
	Rtype    string `json:"rtype" cql:"rtype"`
	Source   string `json:"source" cql:"source"`
	Oneclick string `json:"oneclick" cql:"oneclick"`
	Rurl     string `json:"url" cql:"url"`
}

type Component struct {
	Id                string          `json:"id"`
	Name              string          `json:"name"`
	Tosca             string          `json:"tosca_type"`
	Inputs            pairs.JsonPairs `json:"inputs"`
	Outputs           pairs.JsonPairs `json:"outputs"`
	Envs              pairs.JsonPairs `json:"envs"`
	Repo              Repo            `json:"repo"`
	Artifacts         *Artifacts      `json:"artifacts"`
	RelatedComponents []string        `json:"related_components"`
	Operations        []*Operations   `json:"operations"`
	Status            string          `json:"status"`
	CreatedAt         string          `json:"created_at"`
}

type ComponentTable struct {
	Id                string   `json:"id" cql:"id"`
	Name              string   `json:"name" cql:"name"`
	Tosca             string   `json:"tosca_type" cql:"tosca_type"`
	Inputs            []string `json:"inputs" cql:"inputs"`
	Outputs           []string `json:"outputs" cql:"outputs"`
	Envs              []string `json:"envs" cql:"envs"`
	Repo              string   `json:"repo" cql:"repo"`
	Artifacts         string   `json:"artifacts" cql:"artifacts"`
	RelatedComponents []string `json:"related_components" cql:"related_components"`
	Operations        []string `json:"operations" cql:"operations"`
	Status            string   `json:"status" cql:"status"`
	CreatedAt         string   `json:"created_at" cql:"created_at"`
}

func (a *Component) String() string {
	if d, err := yaml.Marshal(a); err != nil {
		return err.Error()
	} else {
		return string(d)
	}
}

/**
**fetch the component json from riak and parse the json to struct
**/
func NewComponent(id string) (*Component, error) {
	c := &ComponentTable{Id: id}
	ops := ldb.Options{
		TableName:   COMPBUCKET,
		Pks:         []string{"Id"},
		Ccms:        []string{},
		Hosts:       meta.MC.Scylla,
		Keyspace:    meta.MC.ScyllaKeyspace,
		PksClauses:  map[string]interface{}{"Id": id},
		CcmsClauses: make(map[string]interface{}),
	}
	if err := ldb.Fetchdb(ops, c); err != nil {
		return nil, err
	}
	com, _ := c.dig()
	return &com, nil
}

//make a box with the details for a provisioner.
func (c *Component) mkBox() (provision.Box, error) {
	bt := provision.Box{
		Id:         c.Id,
		Level:      provision.BoxSome,
		Name:       c.Name,
		DomainName: c.domain(),
		Envs:       c.envs(),
		Tosca:      c.Tosca,
		Commit:     "",
		Provider:   c.provider(),
		PublicIp:   c.publicIp(),
	}
 fmt.Println("***********Component*********")
fmt.Println(c)
fmt.Println("_______",c.Repo)
	if &c.Repo != nil {
		bt.Repo = &repository.Repo{
			Type:     c.Repo.Rtype,
			Source:   c.Repo.Source,
			OneClick: c.withOneClick(),
			URL:      c.Repo.Rurl,
		}
		// if bt.Repo.URL == "" && bt.Repo.Source == VERTICE && bt.Repo.Type == IMAGE {
		// 	bt.Repo.URL = strings.Split(c.Tosca,".")[2]
		// }

		bt.Repo.Hook = BuildHook(c.Operations, repository.CIHOOK)
	}
	return bt, nil
}

func (c *Component) SetStatus(status utils.Status) error {
	LastStatusUpdate := time.Now().Local().Format(time.RFC822)
	m := make(map[string][]string, 2)
	m["lastsuccessstatusupdate"] = []string{LastStatusUpdate}
	m["status"] = []string{status.String()}
	c.Inputs.NukeAndSet(m) //just nuke the matching output key:

	update_fields := make(map[string]interface{})
	update_fields["Inputs"] = c.Inputs.ToString()
	update_fields["Status"] = status.String()
	ops := ldb.Options{
		TableName:   COMPBUCKET,
		Pks:         []string{"Id"},
		Ccms:        []string{},
		Hosts:       meta.MC.Scylla,
		Keyspace:    meta.MC.ScyllaKeyspace,
		PksClauses:  map[string]interface{}{"Id": c.Id},
		CcmsClauses: make(map[string]interface{}),
	}
	if err := ldb.Updatedb(ops, update_fields); err != nil {
		return err
	}
	return nil
}

/*func (c *Component) UpdateOpsRun(opsRan upgrade.OperationsRan) error {
	mutatedOps := make([]*upgrade.Operation, 0, len(opsRan))

	for _, o := range opsRan {
		mutatedOps = append(mutatedOps, o.Raw)
	}
	c.Operations = mutatedOps

	if err := db.Store(COMPBUCKET, c.Id, c); err != nil {
		return err
	}
	return nil
}*/

func (c *Component) Delete(compid string) {
	ops := ldb.Options{
		TableName:   COMPBUCKET,
		Pks:         []string{"id"},
		Ccms:        []string{},
		Hosts:       meta.MC.Scylla,
		Keyspace:    meta.MC.ScyllaKeyspace,
		PksClauses:  map[string]interface{}{"id": compid},
		CcmsClauses: make(map[string]interface{}),
	}
	if err := ldb.Deletedb(ops, ComponentTable{}); err != nil {
		return
	}
}

func (c *Component) setDeployData(dd DeployData) error {
	/*c.Inputs = append(c.Inputs, utils.NewJsonPair("lastsuccessstatusupdate", ""))
	c.Inputs = append(c.Inputs, utils.NewJsonPair("status", ""))

	if err := db.Store(COMPBUCKET, c.Id, c); err != nil {
		return err
	}*/
	return nil

}

func (a *ComponentTable) dig() (Component, error) {
	asm := Component{}
	asm.Id = a.Id
	asm.Name = a.Name
	asm.Tosca = a.Tosca
	asm.Inputs = a.getInputs()
	asm.Outputs = a.getOutputs()
	asm.Envs = a.getEnvs()
	asm.Repo = a.getRepo()
	asm.Artifacts = a.getArtifacts()
	asm.RelatedComponents = a.RelatedComponents
	asm.Operations = a.getOperations()
	asm.Status = a.Status
	asm.CreatedAt = a.CreatedAt
	return asm, nil
}

func (c *Component) domain() string {
	return c.Inputs.Match(DOMAIN)
}

func (c *Component) provider() string {
	return c.Inputs.Match(utils.PROVIDER)
}

func (c *Component) publicIp() string {
	return c.Outputs.Match(PUBLICIPV4)
}

func (c *Component) withOneClick() bool {
	return (strings.TrimSpace(c.Repo.Oneclick) == TRUE)
}

//all the variables in the inputs shall be treated as ENV.
//we can use a filtered approach as well.
func (c *Component) envs() []bind.EnvVar {
	envs := make([]bind.EnvVar, 0, len(c.Envs))
	for _, i := range c.Envs {
		envs = append(envs, bind.EnvVar{Name: i.K, Value: i.V})
	}
	return envs
}

func (a *ComponentTable) getInputs() pairs.JsonPairs {
	keys := make([]*pairs.JsonPair, 0)
	for _, in := range a.Inputs {
		inputs := pairs.JsonPair{}
		parseStringToStruct(in, &inputs)
		keys = append(keys, &inputs)
	}
	return keys
}

func (a *ComponentTable) getOutputs() pairs.JsonPairs {
	keys := make([]*pairs.JsonPair, 0)
	for _, in := range a.Outputs {
		outputs := pairs.JsonPair{}
		parseStringToStruct(in, &outputs)
		keys = append(keys, &outputs)
	}
	return keys
}

func (a *ComponentTable) getEnvs() pairs.JsonPairs {
	keys := make([]*pairs.JsonPair, 0)
	for _, in := range a.Envs {
		outputs := pairs.JsonPair{}
		parseStringToStruct(in, &outputs)
		keys = append(keys, &outputs)
	}
	return keys
}

func (a *ComponentTable) getRepo() Repo {
	outputs := Repo{}
	parseStringToStruct(a.Repo, &outputs)
	return outputs
}

func (a *ComponentTable) getArtifacts() *Artifacts {
	outputs := Artifacts{}
	parseStringToStruct(a.Artifacts, &outputs)
	return &outputs
}

func (a *ComponentTable) getOperations() []*Operations {
	keys := make([]*Operations, 0)
	for _, in := range a.Operations {
		p := Operations{}
		parseStringToStruct(in, &p)
		keys = append(keys, &p)
	}
	return keys
}
<commit_msg>remove prints<commit_after>/*
** Copyright [2013-2016] [Megam Systems]
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
 */
package carton

import (
	ldb "github.com/megamsys/libgo/db"
	"github.com/megamsys/libgo/pairs"
	"github.com/megamsys/libgo/utils"
	"github.com/megamsys/vertice/carton/bind"
	"github.com/megamsys/vertice/meta"
	"github.com/megamsys/vertice/provision"
	"github.com/megamsys/vertice/repository"
	"gopkg.in/yaml.v2"
	"strings"
	"time"
)

const (
	DOMAIN        = "domain"
	PUBLICIPV4    = "publicipv4"
	PRIVATEIPV4   = "privateipv4"
	COMPBUCKET    = "components"
	IMAGE_VERSION = "version"
	ONECLICK      = "oneclick"
	HOSTIP        = "hostip"
  TRUE          = "true"
)

type Artifacts struct {
	Type         string          `json:"artifact_type" cql:"type"`
	Content      string          `json:"content" cql:"content"`
	Requirements pairs.JsonPairs `json:"requirements" cql:"requirements"`
}

/* Repository represents a repository managed by the manager. */
type Repo struct {
	Rtype    string `json:"rtype" cql:"rtype"`
	Source   string `json:"source" cql:"source"`
	Oneclick string `json:"oneclick" cql:"oneclick"`
	Rurl     string `json:"url" cql:"url"`
}

type Component struct {
	Id                string          `json:"id"`
	Name              string          `json:"name"`
	Tosca             string          `json:"tosca_type"`
	Inputs            pairs.JsonPairs `json:"inputs"`
	Outputs           pairs.JsonPairs `json:"outputs"`
	Envs              pairs.JsonPairs `json:"envs"`
	Repo              Repo            `json:"repo"`
	Artifacts         *Artifacts      `json:"artifacts"`
	RelatedComponents []string        `json:"related_components"`
	Operations        []*Operations   `json:"operations"`
	Status            string          `json:"status"`
	CreatedAt         string          `json:"created_at"`
}

type ComponentTable struct {
	Id                string   `json:"id" cql:"id"`
	Name              string   `json:"name" cql:"name"`
	Tosca             string   `json:"tosca_type" cql:"tosca_type"`
	Inputs            []string `json:"inputs" cql:"inputs"`
	Outputs           []string `json:"outputs" cql:"outputs"`
	Envs              []string `json:"envs" cql:"envs"`
	Repo              string   `json:"repo" cql:"repo"`
	Artifacts         string   `json:"artifacts" cql:"artifacts"`
	RelatedComponents []string `json:"related_components" cql:"related_components"`
	Operations        []string `json:"operations" cql:"operations"`
	Status            string   `json:"status" cql:"status"`
	CreatedAt         string   `json:"created_at" cql:"created_at"`
}

func (a *Component) String() string {
	if d, err := yaml.Marshal(a); err != nil {
		return err.Error()
	} else {
		return string(d)
	}
}

/**
**fetch the component json from riak and parse the json to struct
**/
func NewComponent(id string) (*Component, error) {
	c := &ComponentTable{Id: id}
	ops := ldb.Options{
		TableName:   COMPBUCKET,
		Pks:         []string{"Id"},
		Ccms:        []string{},
		Hosts:       meta.MC.Scylla,
		Keyspace:    meta.MC.ScyllaKeyspace,
		PksClauses:  map[string]interface{}{"Id": id},
		CcmsClauses: make(map[string]interface{}),
	}
	if err := ldb.Fetchdb(ops, c); err != nil {
		return nil, err
	}
	com, _ := c.dig()
	return &com, nil
}

//make a box with the details for a provisioner.
func (c *Component) mkBox() (provision.Box, error) {
	bt := provision.Box{
		Id:         c.Id,
		Level:      provision.BoxSome,
		Name:       c.Name,
		DomainName: c.domain(),
		Envs:       c.envs(),
		Tosca:      c.Tosca,
		Commit:     "",
		Provider:   c.provider(),
		PublicIp:   c.publicIp(),
	}
	if &c.Repo != nil {
		bt.Repo = &repository.Repo{
			Type:     c.Repo.Rtype,
			Source:   c.Repo.Source,
			OneClick: c.withOneClick(),
			URL:      c.Repo.Rurl,
		}
		// if bt.Repo.URL == "" && bt.Repo.Source == VERTICE && bt.Repo.Type == IMAGE {
		// 	bt.Repo.URL = strings.Split(c.Tosca,".")[2]
		// }

		bt.Repo.Hook = BuildHook(c.Operations, repository.CIHOOK)
	}
	return bt, nil
}

func (c *Component) SetStatus(status utils.Status) error {
	LastStatusUpdate := time.Now().Local().Format(time.RFC822)
	m := make(map[string][]string, 2)
	m["lastsuccessstatusupdate"] = []string{LastStatusUpdate}
	m["status"] = []string{status.String()}
	c.Inputs.NukeAndSet(m) //just nuke the matching output key:

	update_fields := make(map[string]interface{})
	update_fields["Inputs"] = c.Inputs.ToString()
	update_fields["Status"] = status.String()
	ops := ldb.Options{
		TableName:   COMPBUCKET,
		Pks:         []string{"Id"},
		Ccms:        []string{},
		Hosts:       meta.MC.Scylla,
		Keyspace:    meta.MC.ScyllaKeyspace,
		PksClauses:  map[string]interface{}{"Id": c.Id},
		CcmsClauses: make(map[string]interface{}),
	}
	if err := ldb.Updatedb(ops, update_fields); err != nil {
		return err
	}
	return nil
}

/*func (c *Component) UpdateOpsRun(opsRan upgrade.OperationsRan) error {
	mutatedOps := make([]*upgrade.Operation, 0, len(opsRan))

	for _, o := range opsRan {
		mutatedOps = append(mutatedOps, o.Raw)
	}
	c.Operations = mutatedOps

	if err := db.Store(COMPBUCKET, c.Id, c); err != nil {
		return err
	}
	return nil
}*/

func (c *Component) Delete(compid string) {
	ops := ldb.Options{
		TableName:   COMPBUCKET,
		Pks:         []string{"id"},
		Ccms:        []string{},
		Hosts:       meta.MC.Scylla,
		Keyspace:    meta.MC.ScyllaKeyspace,
		PksClauses:  map[string]interface{}{"id": compid},
		CcmsClauses: make(map[string]interface{}),
	}
	if err := ldb.Deletedb(ops, ComponentTable{}); err != nil {
		return
	}
}

func (c *Component) setDeployData(dd DeployData) error {
	/*c.Inputs = append(c.Inputs, utils.NewJsonPair("lastsuccessstatusupdate", ""))
	c.Inputs = append(c.Inputs, utils.NewJsonPair("status", ""))

	if err := db.Store(COMPBUCKET, c.Id, c); err != nil {
		return err
	}*/
	return nil

}

func (a *ComponentTable) dig() (Component, error) {
	asm := Component{}
	asm.Id = a.Id
	asm.Name = a.Name
	asm.Tosca = a.Tosca
	asm.Inputs = a.getInputs()
	asm.Outputs = a.getOutputs()
	asm.Envs = a.getEnvs()
	asm.Repo = a.getRepo()
	asm.Artifacts = a.getArtifacts()
	asm.RelatedComponents = a.RelatedComponents
	asm.Operations = a.getOperations()
	asm.Status = a.Status
	asm.CreatedAt = a.CreatedAt
	return asm, nil
}

func (c *Component) domain() string {
	return c.Inputs.Match(DOMAIN)
}

func (c *Component) provider() string {
	return c.Inputs.Match(utils.PROVIDER)
}

func (c *Component) publicIp() string {
	return c.Outputs.Match(PUBLICIPV4)
}

func (c *Component) withOneClick() bool {
	return (strings.TrimSpace(c.Repo.Oneclick) == TRUE)
}

//all the variables in the inputs shall be treated as ENV.
//we can use a filtered approach as well.
func (c *Component) envs() []bind.EnvVar {
	envs := make([]bind.EnvVar, 0, len(c.Envs))
	for _, i := range c.Envs {
		envs = append(envs, bind.EnvVar{Name: i.K, Value: i.V})
	}
	return envs
}

func (a *ComponentTable) getInputs() pairs.JsonPairs {
	keys := make([]*pairs.JsonPair, 0)
	for _, in := range a.Inputs {
		inputs := pairs.JsonPair{}
		parseStringToStruct(in, &inputs)
		keys = append(keys, &inputs)
	}
	return keys
}

func (a *ComponentTable) getOutputs() pairs.JsonPairs {
	keys := make([]*pairs.JsonPair, 0)
	for _, in := range a.Outputs {
		outputs := pairs.JsonPair{}
		parseStringToStruct(in, &outputs)
		keys = append(keys, &outputs)
	}
	return keys
}

func (a *ComponentTable) getEnvs() pairs.JsonPairs {
	keys := make([]*pairs.JsonPair, 0)
	for _, in := range a.Envs {
		outputs := pairs.JsonPair{}
		parseStringToStruct(in, &outputs)
		keys = append(keys, &outputs)
	}
	return keys
}

func (a *ComponentTable) getRepo() Repo {
	outputs := Repo{}
	parseStringToStruct(a.Repo, &outputs)
	return outputs
}

func (a *ComponentTable) getArtifacts() *Artifacts {
	outputs := Artifacts{}
	parseStringToStruct(a.Artifacts, &outputs)
	return &outputs
}

func (a *ComponentTable) getOperations() []*Operations {
	keys := make([]*Operations, 0)
	for _, in := range a.Operations {
		p := Operations{}
		parseStringToStruct(in, &p)
		keys = append(keys, &p)
	}
	return keys
}
<|endoftext|>


<commit_before>package kafkafs

import (
	"path/filepath"
	"log"
	"strconv"
	"strings"
	"regexp"
	"time"
	"fmt"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"
)

// Implements a FUSE filesystem backed by a Kafka installation.
//
// This version is read only, so it cannot post to topics, only read
// from them.
type KafkaRoFs struct {
	pathfs.FileSystem

	KafkaClient KafkaClient
	userFiles map[string]bool
}

type parsedPath struct {
    IsValid bool
	IsRoot bool
	Topic string
	Partition int32
	Offset int64
}

func (fs *KafkaRoFs) parseAndValidate(name string) (parsedPath, error) {
	parsed := parsedPath{IsValid: true, IsRoot: false, Topic: "", Partition: -1,
			Offset: -1}
	slashed := filepath.ToSlash(name)
	re := regexp.MustCompile("/{2,}")
	normal := re.ReplaceAllString(slashed, "/")
	if normal == "" {
		parsed.IsRoot = true
		return parsed, nil
	}

	splits := strings.Split(normal, "/")

	if len(splits) > 3 {
		parsed.IsValid = false
		return parsed, nil
	}

	if len(splits) >= 1 {
		maybeTopic := splits[0]
		isTop, err := fs.isTopic(maybeTopic)
		if err != nil {
			return parsed, err
		}
		if !isTop {
			parsed.IsValid = false
			return parsed, nil
		}
		parsed.Topic = maybeTopic
	}

	if len(splits) >= 2 {
		maybePartition := splits[1]
		isPart, err := fs.isPartition(parsed.Topic, maybePartition)
		if err != nil {
			return parsed, err
		}
		if !isPart {
			parsed.IsValid = false
			return parsed, nil
		}
		// this should always succeed if isPartition returned true.
		partition64, _ := strconv.ParseInt(maybePartition, 10, 32)
		parsed.Partition = int32(partition64)
	}

	if len(splits) == 3 {
		maybeOffset := splits[2]
		offset, err := strconv.ParseInt(maybeOffset, 10, 64)
		if err != nil {
			parsed.IsValid = false
			return parsed, nil
		}
		parsed.Offset = offset
	}

	return parsed, nil
}

func (fs *KafkaRoFs) isTopic(maybeTopic string) (bool, error) {
	topics, err := fs.KafkaClient.GetTopics()
	if err != nil {
		return false, err
	}

	for _, topic := range topics {
		if topic == maybeTopic {
			return true, nil
		}
	}

	return false, nil
}

func (fs *KafkaRoFs) isPartition(topic string, maybePartition string) (bool, error) {
	maybePartition64, err := strconv.ParseInt(maybePartition, 10, 32)
	if err != nil {
		return false, nil
	}
	maybePartition32 := int32(maybePartition64)
	partitions, err := fs.KafkaClient.GetPartitions(topic)
	if err != nil {
		return false, err
	}

	for _, partition := range partitions {
		if partition == maybePartition32 {
			return true, nil
		}
	}

	return false, nil
}

func (fs *KafkaRoFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr,
	fuse.Status) {
	log.Printf("GetAttr name: %s", name)

	parsed, err := fs.parseAndValidate(name)

	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid {
		return nil, fuse.ENOENT
	}

	switch {
	// root or a topic
	case parsed.IsRoot, parsed.Offset == -1 && parsed.Partition == -1:
		return &fuse.Attr{Mode: fuse.S_IFDIR | 0500}, fuse.OK
    // partition
	case parsed.Offset == -1:
		return &fuse.Attr{Mode: fuse.S_IFDIR | 0700}, fuse.OK
    // offset / msg
    case true:
		msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return nil, fuse.EIO
		}
		if msgBytes == nil {
			return nil, fuse.ENOENT
		}
		fs.userFiles[name] = true
		return &fuse.Attr{Mode: fuse.S_IFREG | 0400,
			Size: uint64(len(msgBytes))}, fuse.OK
	}

	return nil, fuse.ENOENT
}

func (fs *KafkaRoFs) OpenDir(name string, context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	log.Printf("OpenDir name: %s", name)

	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid {
		return nil, fuse.ENOENT
	}

	switch {
	case parsed.IsRoot:
		return fs.openRoot(context)
	case parsed.Partition == -1:
		return fs.openTopic(parsed.Topic, context)
	case parsed.Offset == -1:
		return fs.openPartition(parsed.Topic, parsed.Partition, context)
	}

	return nil, fuse.ENOENT
}

func (fs *KafkaRoFs) openPartition(topic string, partition int32,
	context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	earliest, next, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return nil, fuse.EIO
	}
	if next == int64(0) {
		// this means an empty partition
		return []fuse.DirEntry{}, fuse.OK
	}

	entries := []fuse.DirEntry{fuse.DirEntry{Name: strconv.FormatInt(earliest, 10),
		Mode: fuse.S_IFREG}}
	if earliest != next - 1 {
		entries = append(entries, fuse.DirEntry{Name: strconv.FormatInt(next - 1, 10),
			Mode: fuse.S_IFREG})
	}

	for path, _ := range fs.userFiles {
		parsed, err := fs.parseAndValidate(path)
		if err != nil {
			log.Panicf("Error was non-nil, bad user path %s", err)
		}
		if parsed.Partition == partition && parsed.Offset != earliest &&
			parsed.Offset != next - 1 {
			entries = append(entries, fuse.DirEntry{
				Name: strconv.FormatInt(parsed.Offset, 10), Mode: fuse.S_IFREG})
		}
	}

	return entries, fuse.OK
}

func (fs *KafkaRoFs) openRoot(context *fuse.Context) ([]fuse.DirEntry, fuse.Status) {
	// root, show all the topics
	var topicDirs []fuse.DirEntry
	topics, err := fs.KafkaClient.GetTopics()

	if err != nil {
		return nil, fuse.EIO
	}

	for _, topic := range topics {
		topicDirs = append(topicDirs, fuse.DirEntry{Name: topic, Mode: fuse.S_IFDIR})
	}

	return topicDirs, fuse.OK
}

func (fs *KafkaRoFs) openTopic(name string, context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	var partitionDirs []fuse.DirEntry
	partitions, err := fs.KafkaClient.GetPartitions(name)

	if err != nil {
		return nil, fuse.EIO
	}

	for _, partition := range partitions {
		partitionDirs = append(partitionDirs,
			fuse.DirEntry{Name: strconv.FormatInt(int64(partition), 10),
				Mode: fuse.S_IFDIR})
	}

	return partitionDirs, fuse.OK
}

func (fs *KafkaRoFs) Open(name string, flags uint32,
	context *fuse.Context) (nodefs.File, fuse.Status) {
	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid || parsed.Offset == -1 {
		return nil, fuse.ENOENT
	}

	if flags & fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}

	msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
		parsed.Offset)
	if err != nil {
		return nil, fuse.EIO
	}

	return nodefs.NewDataFile(msgBytes), fuse.OK
}

func (fs *KafkaRoFs) Mknod(name string, mode uint32, dev uint32,
	context *fuse.Context) fuse.Status {

	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return fuse.EIO
	}

	if !parsed.IsValid || parsed.Offset == -1 {
		return fuse.ENOENT
	}

	// we don't want to block waiting on an offset that is in the past
	// and no longer available from kafka

	nE := func() (bool, error) {
		return fs.offsetNotExpired(parsed.Topic, parsed.Partition, parsed.Offset)
	}

	for notExpired, err := nE(); notExpired; notExpired, err = nE() {
		if err != nil {
			return fuse.EIO
		}

		isFuture, err := fs.offsetIsFuture(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		if isFuture {
			time.Sleep(1 * time.Second)
			continue
		}

		msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		if msgBytes == nil {
			// this shouldn't happen unless the messages are expiring
			// very fast, but who knows
			return fuse.ENOENT
		}

		err = fs.addUserFile(parsed.Topic, parsed.Partition, parsed.Offset, msgBytes)
		if err != nil {
			return fuse.EIO
		}

		return fuse.OK
	}

	return fuse.ENOENT
}

func (fs *KafkaRoFs) addUserFile(topic string, partition int32, offset int64,
	msgBytes []byte) error {
	path := fmt.Sprintf("%s/%d/%d", topic, partition, offset)
	fs.userFiles[path] = true
	return nil
}

func (fs *KafkaRoFs) offsetNotExpired(topic string, partition int32,
	offset int64) (bool, error) {
	earliest, _, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return false, err
	}
	return offset >= earliest, nil
}

func (fs *KafkaRoFs) offsetIsFuture(topic string, partition int32,
	offset int64) (bool, error) {
	_, next, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return false, err
	}
	return offset >= next, nil
}

// just pretend we set the times to keep touch happy
func (fs *KafkaRoFs) Utimens(name string, Atime *time.Time, Mtime *time.Time,
	context *fuse.Context) fuse.Status {
	return fuse.OK
}

func NewKafkaRoFs(kClient KafkaClient) *KafkaRoFs {
	return &KafkaRoFs{FileSystem: pathfs.NewDefaultFileSystem(), KafkaClient: kClient,
		userFiles: make(map[string]bool)}
}

func (fs *KafkaRoFs) Unlink(name string, context *fuse.Context) fuse.Status {
	_, ok := fs.userFiles[name]
	if !ok {
		return fuse.EPERM
	}
	delete(fs.userFiles, name)
	return fuse.OK
}
<commit_msg>Tabs.<commit_after>package kafkafs

import (
	"path/filepath"
	"log"
	"strconv"
	"strings"
	"regexp"
	"time"
	"fmt"

	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"github.com/hanwen/go-fuse/fuse/pathfs"
)

// Implements a FUSE filesystem backed by a Kafka installation.
//
// This version is read only, so it cannot post to topics, only read
// from them.
type KafkaRoFs struct {
	pathfs.FileSystem

	KafkaClient KafkaClient
	userFiles map[string]bool
}

type parsedPath struct {
	IsValid bool
	IsRoot bool
	Topic string
	Partition int32
	Offset int64
}

func (fs *KafkaRoFs) parseAndValidate(name string) (parsedPath, error) {
	parsed := parsedPath{IsValid: true, IsRoot: false, Topic: "", Partition: -1,
			Offset: -1}
	slashed := filepath.ToSlash(name)
	re := regexp.MustCompile("/{2,}")
	normal := re.ReplaceAllString(slashed, "/")
	if normal == "" {
		parsed.IsRoot = true
		return parsed, nil
	}

	splits := strings.Split(normal, "/")

	if len(splits) > 3 {
		parsed.IsValid = false
		return parsed, nil
	}

	if len(splits) >= 1 {
		maybeTopic := splits[0]
		isTop, err := fs.isTopic(maybeTopic)
		if err != nil {
			return parsed, err
		}
		if !isTop {
			parsed.IsValid = false
			return parsed, nil
		}
		parsed.Topic = maybeTopic
	}

	if len(splits) >= 2 {
		maybePartition := splits[1]
		isPart, err := fs.isPartition(parsed.Topic, maybePartition)
		if err != nil {
			return parsed, err
		}
		if !isPart {
			parsed.IsValid = false
			return parsed, nil
		}
		// this should always succeed if isPartition returned true.
		partition64, _ := strconv.ParseInt(maybePartition, 10, 32)
		parsed.Partition = int32(partition64)
	}

	if len(splits) == 3 {
		maybeOffset := splits[2]
		offset, err := strconv.ParseInt(maybeOffset, 10, 64)
		if err != nil {
			parsed.IsValid = false
			return parsed, nil
		}
		parsed.Offset = offset
	}

	return parsed, nil
}

func (fs *KafkaRoFs) isTopic(maybeTopic string) (bool, error) {
	topics, err := fs.KafkaClient.GetTopics()
	if err != nil {
		return false, err
	}

	for _, topic := range topics {
		if topic == maybeTopic {
			return true, nil
		}
	}

	return false, nil
}

func (fs *KafkaRoFs) isPartition(topic string, maybePartition string) (bool, error) {
	maybePartition64, err := strconv.ParseInt(maybePartition, 10, 32)
	if err != nil {
		return false, nil
	}
	maybePartition32 := int32(maybePartition64)
	partitions, err := fs.KafkaClient.GetPartitions(topic)
	if err != nil {
		return false, err
	}

	for _, partition := range partitions {
		if partition == maybePartition32 {
			return true, nil
		}
	}

	return false, nil
}

func (fs *KafkaRoFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr,
	fuse.Status) {
	log.Printf("GetAttr name: %s", name)

	parsed, err := fs.parseAndValidate(name)

	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid {
		return nil, fuse.ENOENT
	}

	switch {
	// root or a topic
	case parsed.IsRoot, parsed.Offset == -1 && parsed.Partition == -1:
		return &fuse.Attr{Mode: fuse.S_IFDIR | 0500}, fuse.OK
	// partition
	case parsed.Offset == -1:
		return &fuse.Attr{Mode: fuse.S_IFDIR | 0700}, fuse.OK
	// offset / msg
	case true:
		msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return nil, fuse.EIO
		}
		if msgBytes == nil {
			return nil, fuse.ENOENT
		}
		fs.userFiles[name] = true
		return &fuse.Attr{Mode: fuse.S_IFREG | 0400,
			Size: uint64(len(msgBytes))}, fuse.OK
	}

	return nil, fuse.ENOENT
}

func (fs *KafkaRoFs) OpenDir(name string, context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	log.Printf("OpenDir name: %s", name)

	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid {
		return nil, fuse.ENOENT
	}

	switch {
	case parsed.IsRoot:
		return fs.openRoot(context)
	case parsed.Partition == -1:
		return fs.openTopic(parsed.Topic, context)
	case parsed.Offset == -1:
		return fs.openPartition(parsed.Topic, parsed.Partition, context)
	}

	return nil, fuse.ENOENT
}

func (fs *KafkaRoFs) openPartition(topic string, partition int32,
	context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	earliest, next, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return nil, fuse.EIO
	}
	if next == int64(0) {
		// this means an empty partition
		return []fuse.DirEntry{}, fuse.OK
	}

	entries := []fuse.DirEntry{fuse.DirEntry{Name: strconv.FormatInt(earliest, 10),
		Mode: fuse.S_IFREG}}
	if earliest != next - 1 {
		entries = append(entries, fuse.DirEntry{Name: strconv.FormatInt(next - 1, 10),
			Mode: fuse.S_IFREG})
	}

	for path, _ := range fs.userFiles {
		parsed, err := fs.parseAndValidate(path)
		if err != nil {
			log.Panicf("Error was non-nil, bad user path %s", err)
		}
		if parsed.Partition == partition && parsed.Offset != earliest &&
			parsed.Offset != next - 1 {
			entries = append(entries, fuse.DirEntry{
				Name: strconv.FormatInt(parsed.Offset, 10), Mode: fuse.S_IFREG})
		}
	}

	return entries, fuse.OK
}

func (fs *KafkaRoFs) openRoot(context *fuse.Context) ([]fuse.DirEntry, fuse.Status) {
	// root, show all the topics
	var topicDirs []fuse.DirEntry
	topics, err := fs.KafkaClient.GetTopics()

	if err != nil {
		return nil, fuse.EIO
	}

	for _, topic := range topics {
		topicDirs = append(topicDirs, fuse.DirEntry{Name: topic, Mode: fuse.S_IFDIR})
	}

	return topicDirs, fuse.OK
}

func (fs *KafkaRoFs) openTopic(name string, context *fuse.Context) ([]fuse.DirEntry,
	fuse.Status) {
	var partitionDirs []fuse.DirEntry
	partitions, err := fs.KafkaClient.GetPartitions(name)

	if err != nil {
		return nil, fuse.EIO
	}

	for _, partition := range partitions {
		partitionDirs = append(partitionDirs,
			fuse.DirEntry{Name: strconv.FormatInt(int64(partition), 10),
				Mode: fuse.S_IFDIR})
	}

	return partitionDirs, fuse.OK
}

func (fs *KafkaRoFs) Open(name string, flags uint32,
	context *fuse.Context) (nodefs.File, fuse.Status) {
	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return nil, fuse.EIO
	}

	if !parsed.IsValid || parsed.Offset == -1 {
		return nil, fuse.ENOENT
	}

	if flags & fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}

	msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
		parsed.Offset)
	if err != nil {
		return nil, fuse.EIO
	}

	return nodefs.NewDataFile(msgBytes), fuse.OK
}

func (fs *KafkaRoFs) Mknod(name string, mode uint32, dev uint32,
	context *fuse.Context) fuse.Status {

	parsed, err := fs.parseAndValidate(name)
	if err != nil {
		return fuse.EIO
	}

	if !parsed.IsValid || parsed.Offset == -1 {
		return fuse.ENOENT
	}

	// we don't want to block waiting on an offset that is in the past
	// and no longer available from kafka

	nE := func() (bool, error) {
		return fs.offsetNotExpired(parsed.Topic, parsed.Partition, parsed.Offset)
	}

	for notExpired, err := nE(); notExpired; notExpired, err = nE() {
		if err != nil {
			return fuse.EIO
		}

		isFuture, err := fs.offsetIsFuture(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		if isFuture {
			time.Sleep(1 * time.Second)
			continue
		}

		msgBytes, err := fs.KafkaClient.GetMessage(parsed.Topic, parsed.Partition,
			parsed.Offset)
		if err != nil {
			return fuse.EIO
		}

		if msgBytes == nil {
			// this shouldn't happen unless the messages are expiring
			// very fast, but who knows
			return fuse.ENOENT
		}

		err = fs.addUserFile(parsed.Topic, parsed.Partition, parsed.Offset, msgBytes)
		if err != nil {
			return fuse.EIO
		}

		return fuse.OK
	}

	return fuse.ENOENT
}

func (fs *KafkaRoFs) addUserFile(topic string, partition int32, offset int64,
	msgBytes []byte) error {
	path := fmt.Sprintf("%s/%d/%d", topic, partition, offset)
	fs.userFiles[path] = true
	return nil
}

func (fs *KafkaRoFs) offsetNotExpired(topic string, partition int32,
	offset int64) (bool, error) {
	earliest, _, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return false, err
	}
	return offset >= earliest, nil
}

func (fs *KafkaRoFs) offsetIsFuture(topic string, partition int32,
	offset int64) (bool, error) {
	_, next, err := fs.KafkaClient.GetBoundingOffsets(topic, partition)
	if err != nil {
		return false, err
	}
	return offset >= next, nil
}

// just pretend we set the times to keep touch happy
func (fs *KafkaRoFs) Utimens(name string, Atime *time.Time, Mtime *time.Time,
	context *fuse.Context) fuse.Status {
	return fuse.OK
}

func NewKafkaRoFs(kClient KafkaClient) *KafkaRoFs {
	return &KafkaRoFs{FileSystem: pathfs.NewDefaultFileSystem(), KafkaClient: kClient,
		userFiles: make(map[string]bool)}
}

func (fs *KafkaRoFs) Unlink(name string, context *fuse.Context) fuse.Status {
	_, ok := fs.userFiles[name]
	if !ok {
		return fuse.EPERM
	}
	delete(fs.userFiles, name)
	return fuse.OK
}
<|endoftext|>