<commit_before>package KsanaDB
import (
    "fmt"
    "log"
    "encoding/json"
    "strconv"
    "time"
    "errors"
    "strings"
)

var prefix = "KSANADBv1\t"

func SetData(data string) *error {
    InputArray, err := ParseDataJson(data)

    if (err != nil) {
        log.Println(err.Error())
        return &err
    }

    for _, hashdata := range InputArray {
        name := ""
        dataPoints := hashdata.Datapoints

        if hashdata.Name == "" {
            continue
        } else {
            name = hashdata.Name
        }

        if dataPoints == nil {
            value, _ := hashdata.Value.Float64()

            if hashdata.Timestamp == nil {
                log.Println(err.Error())
                continue
            }
            timestamp, err := hashdata.Timestamp.Int64()

            if err != nil {
                log.Println(err.Error())
                continue
            }

            element := make( map[string]interface{})
            keyname, offset := generateTimeSeriesData(prefix, name , timestamp)
            tagSeq := getTagSeq(hashdata.Tags, prefix, name)

            element["timestamp"] = strconv.FormatInt(timestamp, 10)
            element["value"] = strconv.FormatFloat(value, 'f', 6, 64)
            element["tags"] = tagSeq

            jstring, _ := json.Marshal(element)
            input := string(jstring[:])
            SetTimeSeries(keyname, fmt.Sprint(input), offset)
        } else {
            inputData := make(map[string][]interface{})
            tagSeq := getTagSeq(hashdata.Tags, prefix, name)

            for _, data := range dataPoints {
                timestamp, errT := data[0].Int64()
                value, errV := data[1].Float64()

                if errT != nil || errV != nil {
                    log.Println(err.Error())
                    continue
                }

                element := make( map[string]interface{})
                element["timestamp"] = strconv.FormatInt(timestamp, 10)
                element["value"] = strconv.FormatFloat(value, 'f', 6, 64)
                element["tags"] = tagSeq

                jstring, _ := json.Marshal(element)
                input := string(jstring[:])
                keyname, offset := generateTimeSeriesData(prefix, name , timestamp)
                inputData[keyname] = append(inputData[keyname], offset)
                inputData[keyname] = append(inputData[keyname], input)
            }

            for k := range inputData {
                _, err := BulkSetTimeSeries(k, inputData[k])
                if err != nil {
                    log.Println(err.Error())
                    continue
                }
            }

            //fmt.Println(inputData)
        }
    }
    return nil
}

func QueryData(q *Query) (string , error) {
    var start int64
    var end int64
    var err error

    if q.Metric.Name == nil {
        return "", errors.New("Need set Metric name")
    }

    tNow := time.Now()

    if q.StartAbsolute == nil {
        if q.StartRelative == nil {
            return "", errors.New("Need set absolute start time")
        } else {
           start, err = getQueryTime(tNow, q.StartRelative.Unit, q.StartRelative.Value)
           if err != nil {
               return "", err
           }
        }
    } else {
        start, err = q.StartAbsolute.Int64()
        if err != nil {
            return "", err
        }
    }

    if q.EndAbsolute == nil {
        if q.EndRelative == nil {
            return "", errors.New("Need set absolute end time")
        } else {
           end, err = getQueryTime(tNow, q.EndRelative.Unit, q.EndRelative.Value)
           if err != nil {
               return "", err
           }
        }
    } else {
        end, err = q.EndAbsolute.Int64()
        if err != nil {
            return "", err
        }
    }

    if start > end {
        return "", errors.New(fmt.Sprintf("end time(%d) little than start time(%d)", end, start))
    }

    tagFilter := []string{}
    for k, v := range(q.Metric.Tags) {
        tagFilter = append(tagFilter, fmt.Sprintf("%s\t%s", k,v))
    }
    groupByTag := q.Metric.GroupBy
    aggreationFunction := q.Metric.Aggregator.Name
    unit := *q.Metric.Aggregator.Sampling.Unit
    timeRange, err := q.Metric.Aggregator.Sampling.Value.Int64()
    if err != nil {
        return "", err
    }

    ret, err :=  QueryTimeSeriesData(*q.Metric.Name, start, end, tagFilter, groupByTag, aggreationFunction, int(timeRange), unit)
    return ret, err
}

func QueryTimeSeriesData(name string, start int64, stop int64, tagFilter []string, groupByTag []string, aggreationFunction string, timeRange int, unit string) (string , error) {

    groupBy := map[string][]string{}
    var reverseHash map[string]string

    if len(groupByTag) > 0 {
        for _,t := range groupByTag {
            tmp := GetMetricsTagSeq(name, t)
            if len(tmp.Seq[t]) > 0 {
                groupBy[t] = tmp.Seq[t]
            }
            reverseHash = tmp.Val
        }
    }

    tagFilterSeq, err := GetFilterSeq(name, tagFilter)
    if err != nil {
        return "", err
    }

    for _, sq := range (tagFilterSeq) {
        if sq == "" {
            return "", errors.New("Filter Tag(s) not Exist")
        }
    }

    rawData, err := queryTimeSeries(prefix , name , start , stop )

    if err != nil {
        return "{}", err
    }

    if len(rawData) == 0 {
        return "{}", nil
    }

    data, err := queryWorker(rawData, start, tagFilterSeq, groupBy, aggreationFunction, unit, timeRange)

    ret, err := generateOutputData(data, reverseHash, name, start, stop, tagFilter, groupByTag, aggreationFunction, timeRange, unit)

    fmt.Print("Find record(s): ")
    fmt.Println(len(rawData))
    return ret, err
}

func GetMetricsTag(name string, target string, keyName string) (map[string][]string)  {
    var data string
    switch target {
        case "All", "TagKey", "TagValue" :
            data = getTags(prefix, name, target, keyName)
      //  default:

    }
    var ret map[string][]string
    json.Unmarshal([]byte(data), &ret)
    return ret
}

func GetMetricsTagSeq(name string, keyName string) (AllTagSeqType)  {
    data := getTags(prefix, name, "TagSeq", keyName)
    var ret AllTagSeqType
    json.Unmarshal([]byte(data), &ret)
    return ret
}

func GetFilterSeq(name string, filterList []string) ([]string, error){
      if len(filterList) == 0 {
          ret := []string{}
          return ret, nil
      }
      d, err := getSeqByKV(prefix, name, filterList)
      return d, err
}

func generateOutputData(result map[string][]map[string]interface{}, reverseHash map[string]string, name string, start int64, stop int64, tagFilter []string, groupByTag []string, aggregationFunction string, timeRange int, unit string) (string , error) {
    resultData := ResultType{}
    resultData.Group = []GroupType{}

    resultData.Name = name
    resultData.GroupBy = groupByTag

    resultData.Start = start
    resultData.End = stop
    resultData.Filter = tagFilter
    resultData.AggregateFunction = aggregationFunction
    resultData.TimeRange = timeRange
    resultData.TimeUnit = unit

    for k,v := range(result) {
       gp := GroupType{}
       gp.Tags = map[string]string{}
       gp.Values = [][]interface{}{}

       ka := strings.Split(k, "\t")
       for _, s := range(ka) {
           tagPair := reverseHash[s]
           tagKV := strings.Split(tagPair, "\t")
           gp.Tags[tagKV[0]] = tagKV[1]
       }

       for _,d := range(v) {
           ele := []interface{}{}
           ele = append(ele, d["timestamp"])
           ele = append(ele, d["value"])
           gp.Values = append(gp.Values, ele)
       }
       resultData.Group = append(resultData.Group, gp)
    }
    jret, err := json.Marshal(resultData)
    return string(jret), err
}
<commit_msg>fix bug and add output function<commit_after>package KsanaDB
import (
    "fmt"
    "log"
    "encoding/json"
    "strconv"
    "time"
    "errors"
    "strings"
)

var prefix = "KSANADBv1\t"

func SetData(data string) *error {
    InputArray, err := ParseDataJson(data)

    if (err != nil) {
        log.Println(err.Error())
        return &err
    }

    for _, hashdata := range InputArray {
        name := ""
        dataPoints := hashdata.Datapoints

        if hashdata.Name == "" {
            continue
        } else {
            name = hashdata.Name
        }

        if dataPoints == nil {
            value, _ := hashdata.Value.Float64()

            if hashdata.Timestamp == nil {
                log.Println(err.Error())
                continue
            }
            timestamp, err := hashdata.Timestamp.Int64()

            if err != nil {
                log.Println(err.Error())
                continue
            }

            element := make( map[string]interface{})
            keyname, offset := generateTimeSeriesData(prefix, name , timestamp)
            tagSeq := getTagSeq(hashdata.Tags, prefix, name)

            element["timestamp"] = strconv.FormatInt(timestamp, 10)
            element["value"] = strconv.FormatFloat(value, 'f', 6, 64)
            element["tags"] = tagSeq

            jstring, _ := json.Marshal(element)
            input := string(jstring[:])
            SetTimeSeries(keyname, fmt.Sprint(input), offset)
        } else {
            inputData := make(map[string][]interface{})
            tagSeq := getTagSeq(hashdata.Tags, prefix, name)

            for _, data := range dataPoints {
                timestamp, errT := data[0].Int64()
                value, errV := data[1].Float64()

                if errT != nil || errV != nil {
                    log.Println(err.Error())
                    continue
                }

                element := make( map[string]interface{})
                element["timestamp"] = strconv.FormatInt(timestamp, 10)
                element["value"] = strconv.FormatFloat(value, 'f', 6, 64)
                element["tags"] = tagSeq

                jstring, _ := json.Marshal(element)
                input := string(jstring[:])
                keyname, offset := generateTimeSeriesData(prefix, name , timestamp)
                inputData[keyname] = append(inputData[keyname], offset)
                inputData[keyname] = append(inputData[keyname], input)
            }

            for k := range inputData {
                _, err := BulkSetTimeSeries(k, inputData[k])
                if err != nil {
                    log.Println(err.Error())
                    continue
                }
            }

            //fmt.Println(inputData)
        }
    }
    return nil
}

func QueryData(q *Query) (string , error) {
    var start int64
    var end int64
    var err error

    if q.Metric.Name == nil {
        return "", errors.New("Need set Metric name")
    }

    tNow := time.Now()

    if q.StartAbsolute == nil {
        if q.StartRelative == nil {
            return "", errors.New("Need set absolute start time")
        } else {
           start, err = getQueryTime(tNow, q.StartRelative.Unit, q.StartRelative.Value)
           if err != nil {
               return "", err
           }
        }
    } else {
        start, err = q.StartAbsolute.Int64()
        if err != nil {
            return "", err
        }
    }

    if q.EndAbsolute == nil {
        if q.EndRelative == nil {
            return "", errors.New("Need set absolute end time")
        } else {
           end, err = getQueryTime(tNow, q.EndRelative.Unit, q.EndRelative.Value)
           if err != nil {
               return "", err
           }
        }
    } else {
        end, err = q.EndAbsolute.Int64()
        if err != nil {
            return "", err
        }
    }

    if start > end {
        return "", errors.New(fmt.Sprintf("end time(%d) little than start time(%d)", end, start))
    }

    tagFilter := []string{}
    for k, v := range(q.Metric.Tags) {
        tagFilter = append(tagFilter, fmt.Sprintf("%s\t%s", k,v))
    }
    groupByTag := q.Metric.GroupBy
    aggreationFunction := q.Metric.Aggregator.Name
    unit := *q.Metric.Aggregator.Sampling.Unit
    timeRange, err := q.Metric.Aggregator.Sampling.Value.Int64()
    if err != nil {
        return "", err
    }

    ret, err :=  QueryTimeSeriesData(*q.Metric.Name, start, end, tagFilter, groupByTag, aggreationFunction, int(timeRange), unit)
    return ret, err
}

func QueryTimeSeriesData(name string, start int64, stop int64, tagFilter []string, groupByTag []string, aggreationFunction string, timeRange int, unit string) (string , error) {

    groupBy := map[string][]string{}
    var reverseHash map[string]string

    if len(groupByTag) > 0 {
        for _,t := range groupByTag {
            tmp := GetMetricsTagSeq(name, t)
            if len(tmp.Seq[t]) > 0 {
                groupBy[t] = tmp.Seq[t]
            } else {
                return "", errors.New("Input Group by tag(s) not Exist")
            }
            reverseHash = tmp.Val
        }
    }

    tagFilterSeq, err := GetFilterSeq(name, tagFilter)
    if err != nil {
        return "", err
    }

    for _, sq := range (tagFilterSeq) {
        if sq == "" {
            return "", errors.New("Input Filter Tag(s) not Exist")
        }
    }

    rawData, err := queryTimeSeries(prefix , name , start , stop )

    if err != nil {
        return "{}", err
    }

    if len(rawData) == 0 {
        return "{}", nil
    }

    data, err := queryWorker(rawData, start, tagFilterSeq, groupBy, aggreationFunction, unit, timeRange)

    ret, err := generateOutputData(data, reverseHash, name, start, stop, tagFilter, groupByTag, aggreationFunction, timeRange, unit)

    fmt.Print("Find record(s): ")
    fmt.Println(len(rawData))
    return ret, err
}

func GetMetricsTag(name string, target string, keyName string) (map[string][]string)  {
    var data string
    switch target {
        case "All", "TagKey", "TagValue" :
            data = getTags(prefix, name, target, keyName)
      //  default:

    }
    var ret map[string][]string
    json.Unmarshal([]byte(data), &ret)
    return ret
}

func GetMetricsTagSeq(name string, keyName string) (AllTagSeqType)  {
    data := getTags(prefix, name, "TagSeq", keyName)
    var ret AllTagSeqType
    json.Unmarshal([]byte(data), &ret)
    return ret
}

func GetFilterSeq(name string, filterList []string) ([]string, error){
      if len(filterList) == 0 {
          ret := []string{}
          return ret, nil
      }
      d, err := getSeqByKV(prefix, name, filterList)
      return d, err
}

func generateOutputData(result map[string][]map[string]interface{}, reverseHash map[string]string, name string, start int64, stop int64, tagFilter []string, groupByTag []string, aggregationFunction string, timeRange int, unit string) (string , error) {
    resultData := ResultType{}
    resultData.Group = []GroupType{}

    resultData.Name = name
    resultData.GroupBy = groupByTag

    resultData.Start = start
    resultData.End = stop
    resultData.Filter = tagFilter
    resultData.AggregateFunction = aggregationFunction
    resultData.TimeRange = timeRange
    resultData.TimeUnit = unit

    for k,v := range(result) {
       gp := GroupType{}
       gp.Tags = map[string]string{}
       gp.Values = [][]interface{}{}

       ka := strings.Split(k, "\t")
       for _, s := range(ka) {
           tagPair := reverseHash[s]
           tagKV := strings.Split(tagPair, "\t")
           gp.Tags[tagKV[0]] = tagKV[1]
       }

       for _,d := range(v) {
           ele := []interface{}{}
           ele = append(ele, d["timestamp"])
           ele = append(ele, d["value"])
           gp.Values = append(gp.Values, ele)
       }
       resultData.Group = append(resultData.Group, gp)
    }
    jret, err := json.Marshal(resultData)
    return string(jret), err
}
<|endoftext|>


<commit_before>package postgresql

import (
	"fmt"
	"strconv"
	"time"

	"github.com/Tecsisa/foulkon/api"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq" //GORM needs to import the lib/pq driver
)

type PostgresRepo struct {
	Dbmap *gorm.DB
}

func InitDb(datasourcename string, idleConns string, maxOpenConns string, connTTL string) (*gorm.DB, error) {
	// connect to db using GORM - github.com/jinzhu/gorm
	db, err := gorm.Open("postgres", datasourcename)
	if err != nil {
		return nil, err
	}

	// construct a gorm DbMap
	idle, err := strconv.Atoi(idleConns)
	if err != nil {
		return nil, fmt.Errorf("Invalid postgresql idleConns param: %v", idleConns)
	}
	maxOpen, err := strconv.Atoi(maxOpenConns)
	if err != nil {
		return nil, fmt.Errorf("Invalid postgresql maxOpenConns param: %v", maxOpenConns)
	}
	ttl, err := strconv.Atoi(connTTL)
	if err != nil {
		return nil, fmt.Errorf("Invalid postgresql connTTL param: %v", connTTL)
	}
	db.DB().SetMaxIdleConns(idle)
	db.DB().SetMaxOpenConns(maxOpen)
	db.DB().SetConnMaxLifetime(time.Duration(ttl) * time.Second)

	// Check connection
	err = db.DB().Ping()
	if err != nil {
		return nil, err
	}

	// Create tables if not exist
	err = db.AutoMigrate(&User{}, &Group{}, &Policy{}, &Statement{}, &GroupUserRelation{}, &GroupPolicyRelation{}, &ProxyResource{}).Error
	if err != nil {
		return nil, err
	}

	return db, nil
}

// User table
type User struct {
	ID         string `gorm:"primary_key"`
	ExternalID string `gorm:"not null;unique"`
	Path       string `gorm:"not null"`
	CreateAt   int64  `gorm:"not null"`
	UpdateAt   int64  `gorm:"not null"`
	Urn        string `gorm:"not null;unique"`
}

// User's table name
func (User) TableName() string {
	return "users"
}

// Group table
type Group struct {
	ID       string `gorm:"primary_key"`
	Name     string `gorm:"not null"`
	Path     string `gorm:"not null"`
	Org      string `gorm:"not null"`
	CreateAt int64  `gorm:"not null"`
	UpdateAt int64  `gorm:"not null"`
	Urn      string `gorm:"not null;unique"`
}

// Group's table name
func (Group) TableName() string {
	return "groups"
}

// Policy table
type Policy struct {
	ID       string `gorm:"primary_key"`
	Name     string `gorm:"not null"`
	Path     string `gorm:"not null"`
	Org      string `gorm:"not null"`
	CreateAt int64  `gorm:"not null"`
	UpdateAt int64  `gorm:"not null"`
	Urn      string `gorm:"not null;unique"`
}

// Policy's table name
func (Policy) TableName() string {
	return "policies"
}

// Statement table
type Statement struct {
	ID        string `gorm:"primary_key"`
	PolicyID  string `gorm:"not null"`
	Effect    string `gorm:"not null"`
	Actions   string `gorm:"not null"`
	Resources string `gorm:"not null"`
}

// Statement's table name
func (Statement) TableName() string {
	return "statements"
}

// Group-Users Relationship
type GroupUserRelation struct {
	UserID   string `gorm:"primary_key"`
	GroupID  string `gorm:"primary_key"`
	CreateAt int64  `gorm:"not null"`
}

// GroupUserRelation's table name
func (GroupUserRelation) TableName() string {
	return "group_user_relations"
}

// Group Policy table
type GroupPolicyRelation struct {
	GroupID  string `gorm:"primary_key"`
	PolicyID string `gorm:"primary_key"`
	CreateAt int64  `gorm:"not null"`
}

// GroupPolicyRelation's table name
func (GroupPolicyRelation) TableName() string {
	return "group_policy_relations"
}

func (u PostgresRepo) OrderByValidColumns(action string) []string {
	switch action {
	case api.USER_ACTION_LIST_USERS:
		return []string{"path", "external_id", "create_at", "update_at", "urn"}
	case api.USER_ACTION_LIST_GROUPS_FOR_USER:
		return []string{"create_at"}
	case api.GROUP_ACTION_LIST_GROUPS:
		return []string{"name", "path", "org", "create_at", "update_at", "urn"}
	case api.GROUP_ACTION_LIST_MEMBERS:
		return []string{"create_at"}
	case api.GROUP_ACTION_LIST_ATTACHED_GROUP_POLICIES:
		return []string{"create_at"}
	case api.POLICY_ACTION_LIST_POLICIES:
		return []string{"name", "path", "org", "create_at", "update_at", "urn"}
	case api.POLICY_ACTION_LIST_ATTACHED_GROUPS:
		return []string{"create_at"}
	default:
		return nil
	}
}

// ProxyResource table
type ProxyResource struct {
	ID     string `gorm:"primary_key"`
	Host   string `gorm:"not null"`
	Url    string `gorm:"not null;unique_index:idx_url_method"`
	Method string `gorm:"not null;unique_index:idx_url_method"`
	Urn    string `gorm:"not null;unique"`
	Action string `gorm:"not null"`
}

// ProxyResource's table name
func (ProxyResource) TableName() string {
	return "proxy_resources"
}
<commit_msg>update resource uk in postgres db<commit_after>package postgresql

import (
	"fmt"
	"strconv"
	"time"

	"github.com/Tecsisa/foulkon/api"
	"github.com/jinzhu/gorm"
	_ "github.com/lib/pq" //GORM needs to import the lib/pq driver
)

type PostgresRepo struct {
	Dbmap *gorm.DB
}

func InitDb(datasourcename string, idleConns string, maxOpenConns string, connTTL string) (*gorm.DB, error) {
	// connect to db using GORM - github.com/jinzhu/gorm
	db, err := gorm.Open("postgres", datasourcename)
	if err != nil {
		return nil, err
	}

	// construct a gorm DbMap
	idle, err := strconv.Atoi(idleConns)
	if err != nil {
		return nil, fmt.Errorf("Invalid postgresql idleConns param: %v", idleConns)
	}
	maxOpen, err := strconv.Atoi(maxOpenConns)
	if err != nil {
		return nil, fmt.Errorf("Invalid postgresql maxOpenConns param: %v", maxOpenConns)
	}
	ttl, err := strconv.Atoi(connTTL)
	if err != nil {
		return nil, fmt.Errorf("Invalid postgresql connTTL param: %v", connTTL)
	}
	db.DB().SetMaxIdleConns(idle)
	db.DB().SetMaxOpenConns(maxOpen)
	db.DB().SetConnMaxLifetime(time.Duration(ttl) * time.Second)

	// Check connection
	err = db.DB().Ping()
	if err != nil {
		return nil, err
	}

	// Create tables if not exist
	err = db.AutoMigrate(&User{}, &Group{}, &Policy{}, &Statement{}, &GroupUserRelation{}, &GroupPolicyRelation{}, &ProxyResource{}).Error
	if err != nil {
		return nil, err
	}

	return db, nil
}

// User table
type User struct {
	ID         string `gorm:"primary_key"`
	ExternalID string `gorm:"not null;unique"`
	Path       string `gorm:"not null"`
	CreateAt   int64  `gorm:"not null"`
	UpdateAt   int64  `gorm:"not null"`
	Urn        string `gorm:"not null;unique"`
}

// User's table name
func (User) TableName() string {
	return "users"
}

// Group table
type Group struct {
	ID       string `gorm:"primary_key"`
	Name     string `gorm:"not null"`
	Path     string `gorm:"not null"`
	Org      string `gorm:"not null"`
	CreateAt int64  `gorm:"not null"`
	UpdateAt int64  `gorm:"not null"`
	Urn      string `gorm:"not null;unique"`
}

// Group's table name
func (Group) TableName() string {
	return "groups"
}

// Policy table
type Policy struct {
	ID       string `gorm:"primary_key"`
	Name     string `gorm:"not null"`
	Path     string `gorm:"not null"`
	Org      string `gorm:"not null"`
	CreateAt int64  `gorm:"not null"`
	UpdateAt int64  `gorm:"not null"`
	Urn      string `gorm:"not null;unique"`
}

// Policy's table name
func (Policy) TableName() string {
	return "policies"
}

// Statement table
type Statement struct {
	ID        string `gorm:"primary_key"`
	PolicyID  string `gorm:"not null"`
	Effect    string `gorm:"not null"`
	Actions   string `gorm:"not null"`
	Resources string `gorm:"not null"`
}

// Statement's table name
func (Statement) TableName() string {
	return "statements"
}

// Group-Users Relationship
type GroupUserRelation struct {
	UserID   string `gorm:"primary_key"`
	GroupID  string `gorm:"primary_key"`
	CreateAt int64  `gorm:"not null"`
}

// GroupUserRelation's table name
func (GroupUserRelation) TableName() string {
	return "group_user_relations"
}

// Group Policy table
type GroupPolicyRelation struct {
	GroupID  string `gorm:"primary_key"`
	PolicyID string `gorm:"primary_key"`
	CreateAt int64  `gorm:"not null"`
}

// GroupPolicyRelation's table name
func (GroupPolicyRelation) TableName() string {
	return "group_policy_relations"
}

func (u PostgresRepo) OrderByValidColumns(action string) []string {
	switch action {
	case api.USER_ACTION_LIST_USERS:
		return []string{"path", "external_id", "create_at", "update_at", "urn"}
	case api.USER_ACTION_LIST_GROUPS_FOR_USER:
		return []string{"create_at"}
	case api.GROUP_ACTION_LIST_GROUPS:
		return []string{"name", "path", "org", "create_at", "update_at", "urn"}
	case api.GROUP_ACTION_LIST_MEMBERS:
		return []string{"create_at"}
	case api.GROUP_ACTION_LIST_ATTACHED_GROUP_POLICIES:
		return []string{"create_at"}
	case api.POLICY_ACTION_LIST_POLICIES:
		return []string{"name", "path", "org", "create_at", "update_at", "urn"}
	case api.POLICY_ACTION_LIST_ATTACHED_GROUPS:
		return []string{"create_at"}
	default:
		return nil
	}
}

// ProxyResource table
type ProxyResource struct {
	ID     string `gorm:"primary_key"`
	Host   string `gorm:"not null;unique_index:idx_resource"`
	Url    string `gorm:"not null;unique_index:idx_resource"`
	Method string `gorm:"not null;unique_index:idx_resource"`
	Urn    string `gorm:"not null;unique_index:idx_resource"`
	Action string `gorm:"not null;unique_index:idx_resource"`
}

// ProxyResource's table name
func (ProxyResource) TableName() string {
	return "proxy_resources"
}
<|endoftext|>


<commit_before>// Copyright (c) 2019 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package intdataplane

import (
	"encoding/binary"
	"fmt"
	"net"

	"golang.org/x/sys/unix"

	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/bpf"
)

type bpfNATManager struct {
	natMap     bpf.Map
	backendMap bpf.Map
}

// struct calico_nat_v4_key {
//    uint32_t addr; // NBO
//    uint16_t port; // HBO
//    uint8_t protocol;
//    uint8_t pad;
// };
const natKeySize = 8

// struct calico_nat_v4_value {
//    uint32_t id;
//    uint32_t count;
// };
const natValueSize = 8

// struct calico_nat_secondary_v4_key {
//   uint32_t id;
//   uint32_t ordinal;
// };
const natBackendKeySize = 8

// struct calico_nat_dest {
//    uint32_t addr;
//    uint16_t port;
//    uint8_t pad[2];
// };
const natBackendValueSize = 8

type NATKey [natKeySize]byte

func NewNATKey(addr net.IP, port uint16, protocol uint8) NATKey {
	var k NATKey
	addr = addr.To4()
	if len(addr) != 4 {
		log.WithField("ip", addr).Panic("Bad IP")
	}
	copy(k[:4], addr)
	binary.LittleEndian.PutUint16(k[4:6], port)
	k[6] = protocol
	return k
}

func (k NATKey) Proto() uint8 {
	return k[6]
}

func (k NATKey) Addr() net.IP {
	return k[:4]
}

func (k NATKey) Port() uint16 {
	return binary.LittleEndian.Uint16(k[4:6])
}

func (k NATKey) String() string {
	return fmt.Sprintf("proto=%v %v:%v", k.Proto(), k.Addr(), k.Port())
}

type NATValue [natValueSize]byte

func NewNATValue(id, count uint32) NATValue {
	var v NATValue
	binary.LittleEndian.PutUint32(v[:4], id)
	binary.LittleEndian.PutUint32(v[4:8], count)
	return v
}

func (v NATValue) ID() uint32 {
	return binary.LittleEndian.Uint32(v[:4])
}

func (v NATValue) Count() uint32 {
	return binary.LittleEndian.Uint32(v[4:8])
}

func (v NATValue) String() string {
	return fmt.Sprintf("NATValue{ID:%d,Count:%d}", v.ID(), v.Count())
}

type NATBackendKey [natBackendKeySize]byte

func NewNATBackendKey(id, ordinal uint32) NATBackendKey {
	var v NATBackendKey
	binary.LittleEndian.PutUint32(v[:4], id)
	binary.LittleEndian.PutUint32(v[4:8], ordinal)
	return v
}

func (v NATBackendKey) ID() uint32 {
	return binary.LittleEndian.Uint32(v[:4])
}

func (v NATBackendKey) Count() uint32 {
	return binary.LittleEndian.Uint32(v[4:8])
}

func (v NATBackendKey) String() string {
	return fmt.Sprintf("NATBackendKey{ID:%d,Ordinal:%d}", v.ID(), v.Count())
}

type NATBackendValue [natBackendValueSize]byte

func NewNATBackendValue(addr net.IP, port uint16) NATBackendValue {
	var k NATBackendValue
	addr = addr.To4()
	if len(addr) != 4 {
		log.WithField("ip", addr).Panic("Bad IP")
	}
	copy(k[:4], addr)
	binary.LittleEndian.PutUint16(k[4:6], port)
	return k
}

func (k NATBackendValue) Addr() net.IP {
	return k[:4]
}

func (k NATBackendValue) Port() uint16 {
	return binary.LittleEndian.Uint16(k[4:6])
}

func (k NATBackendValue) String() string {
	return fmt.Sprintf("%v:%v", k.Addr(), k.Port())
}

func newBPFNATManager() *bpfNATManager {
	return &bpfNATManager{
		natMap:     NATMap(),
		backendMap: BackendMap(),
	}
}

func NATMap() bpf.Map {
	return bpf.NewPinnedMap(
		"cali_nat_v4",
		"/sys/fs/bpf/tc/globals/cali_nat_v4",
		"hash",
		natKeySize,
		natValueSize,
		511000,
		unix.BPF_F_NO_PREALLOC)
}

func BackendMap() bpf.Map {
	return bpf.NewPinnedMap(
		"cali_natbe_v4",
		"/sys/fs/bpf/tc/globals/cali_natbe_v4",
		"hash",
		natKeySize,
		natValueSize,
		510000,
		unix.BPF_F_NO_PREALLOC)
}

func (m *bpfNATManager) OnUpdate(msg interface{}) {
}

func (m *bpfNATManager) CompleteDeferredWork() error {
	err := m.natMap.EnsureExists()
	if err != nil {
		log.WithError(err).Panic("Failed to create NAT map")
	}
	err = m.backendMap.EnsureExists()
	if err != nil {
		log.WithError(err).Panic("Failed to create NAT map")
	}

	bk := NewNATBackendKey(123, 0)
	wlAddr := net.ParseIP("10.65.0.10")
	bv := NewNATBackendValue(wlAddr, 8055)
	err = m.backendMap.Update(bk[:], bv[:])
	if err != nil {
		log.WithError(err).Panic("Failed to update backend NAT map.")
	}

	natAddr := net.ParseIP("10.96.0.10")
	nk := NewNATKey(natAddr, 80, 6)
	nv := NewNATValue(123, 1)
	err = m.natMap.Update(nk[:], nv[:])
	if err != nil {
		log.WithError(err).Panic("Failed to update NAT map.")
	}

	log.Info("Updated NAT mappings")

	// Make sure we get retried.
	return nil
}
<commit_msg>NATKey and NATBackendValue String() fix<commit_after>// Copyright (c) 2019 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package intdataplane

import (
	"encoding/binary"
	"fmt"
	"net"

	"golang.org/x/sys/unix"

	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/bpf"
)

type bpfNATManager struct {
	natMap     bpf.Map
	backendMap bpf.Map
}

// struct calico_nat_v4_key {
//    uint32_t addr; // NBO
//    uint16_t port; // HBO
//    uint8_t protocol;
//    uint8_t pad;
// };
const natKeySize = 8

// struct calico_nat_v4_value {
//    uint32_t id;
//    uint32_t count;
// };
const natValueSize = 8

// struct calico_nat_secondary_v4_key {
//   uint32_t id;
//   uint32_t ordinal;
// };
const natBackendKeySize = 8

// struct calico_nat_dest {
//    uint32_t addr;
//    uint16_t port;
//    uint8_t pad[2];
// };
const natBackendValueSize = 8

type NATKey [natKeySize]byte

func NewNATKey(addr net.IP, port uint16, protocol uint8) NATKey {
	var k NATKey
	addr = addr.To4()
	if len(addr) != 4 {
		log.WithField("ip", addr).Panic("Bad IP")
	}
	copy(k[:4], addr)
	binary.LittleEndian.PutUint16(k[4:6], port)
	k[6] = protocol
	return k
}

func (k NATKey) Proto() uint8 {
	return k[6]
}

func (k NATKey) Addr() net.IP {
	return k[:4]
}

func (k NATKey) Port() uint16 {
	return binary.LittleEndian.Uint16(k[4:6])
}

func (k NATKey) String() string {
	return fmt.Sprintf("NATKey{Proto:%v Addr:%v Port:%v}", k.Proto(), k.Addr(), k.Port())
}

type NATValue [natValueSize]byte

func NewNATValue(id, count uint32) NATValue {
	var v NATValue
	binary.LittleEndian.PutUint32(v[:4], id)
	binary.LittleEndian.PutUint32(v[4:8], count)
	return v
}

func (v NATValue) ID() uint32 {
	return binary.LittleEndian.Uint32(v[:4])
}

func (v NATValue) Count() uint32 {
	return binary.LittleEndian.Uint32(v[4:8])
}

func (v NATValue) String() string {
	return fmt.Sprintf("NATValue{ID:%d,Count:%d}", v.ID(), v.Count())
}

type NATBackendKey [natBackendKeySize]byte

func NewNATBackendKey(id, ordinal uint32) NATBackendKey {
	var v NATBackendKey
	binary.LittleEndian.PutUint32(v[:4], id)
	binary.LittleEndian.PutUint32(v[4:8], ordinal)
	return v
}

func (v NATBackendKey) ID() uint32 {
	return binary.LittleEndian.Uint32(v[:4])
}

func (v NATBackendKey) Count() uint32 {
	return binary.LittleEndian.Uint32(v[4:8])
}

func (v NATBackendKey) String() string {
	return fmt.Sprintf("NATBackendKey{ID:%d,Ordinal:%d}", v.ID(), v.Count())
}

type NATBackendValue [natBackendValueSize]byte

func NewNATBackendValue(addr net.IP, port uint16) NATBackendValue {
	var k NATBackendValue
	addr = addr.To4()
	if len(addr) != 4 {
		log.WithField("ip", addr).Panic("Bad IP")
	}
	copy(k[:4], addr)
	binary.LittleEndian.PutUint16(k[4:6], port)
	return k
}

func (k NATBackendValue) Addr() net.IP {
	return k[:4]
}

func (k NATBackendValue) Port() uint16 {
	return binary.LittleEndian.Uint16(k[4:6])
}

func (k NATBackendValue) String() string {
	return fmt.Sprintf("NATBackendValue{Addr:%v Port:%v}", k.Addr(), k.Port())
}

func newBPFNATManager() *bpfNATManager {
	return &bpfNATManager{
		natMap:     NATMap(),
		backendMap: BackendMap(),
	}
}

func NATMap() bpf.Map {
	return bpf.NewPinnedMap(
		"cali_nat_v4",
		"/sys/fs/bpf/tc/globals/cali_nat_v4",
		"hash",
		natKeySize,
		natValueSize,
		511000,
		unix.BPF_F_NO_PREALLOC)
}

func BackendMap() bpf.Map {
	return bpf.NewPinnedMap(
		"cali_natbe_v4",
		"/sys/fs/bpf/tc/globals/cali_natbe_v4",
		"hash",
		natKeySize,
		natValueSize,
		510000,
		unix.BPF_F_NO_PREALLOC)
}

func (m *bpfNATManager) OnUpdate(msg interface{}) {
}

func (m *bpfNATManager) CompleteDeferredWork() error {
	err := m.natMap.EnsureExists()
	if err != nil {
		log.WithError(err).Panic("Failed to create NAT map")
	}
	err = m.backendMap.EnsureExists()
	if err != nil {
		log.WithError(err).Panic("Failed to create NAT map")
	}

	bk := NewNATBackendKey(123, 0)
	wlAddr := net.ParseIP("10.65.0.10")
	bv := NewNATBackendValue(wlAddr, 8055)
	err = m.backendMap.Update(bk[:], bv[:])
	if err != nil {
		log.WithError(err).Panic("Failed to update backend NAT map.")
	}

	natAddr := net.ParseIP("10.96.0.10")
	nk := NewNATKey(natAddr, 80, 6)
	nv := NewNATValue(123, 1)
	err = m.natMap.Update(nk[:], nv[:])
	if err != nil {
		log.WithError(err).Panic("Failed to update NAT map.")
	}

	log.Info("Updated NAT mappings")

	// Make sure we get retried.
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package main

import (
	"flag"
	"os"
	"os/signal"

	"github.com/maruel/subcommands"
	"golang.org/x/net/context"

	"github.com/luci/luci-go/client/authcli"
	"github.com/luci/luci-go/common/auth"
	"github.com/luci/luci-go/common/cli"
	log "github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/logging/gologger"
)

type application struct {
	cli.Application

	authFlags  authcli.Flags
	layoutPath string
	workers    int

	layout deployLayout
	tools  tools
}

func (a *application) addFlags(fs *flag.FlagSet) {
	fs.StringVar(&a.layoutPath, "layout", a.layoutPath,
		"Path to the base layout file.")
	fs.IntVar(&a.workers, "workers", 0,
		"Maximum number of workers to use. If <= 0, no limit will be applied.")
}

func (a *application) runWork(c context.Context, fn func(*work) error) error {
	return runWork(c, a.workers, &a.tools, fn)
}

func mainImpl(args []string) int {
	authOptions := auth.Options{
		Scopes: []string{
			"https://www.googleapis.com/auth/cloud-platform",
		},
	}

	c := gologger.StdConfig.Use(context.Background())
	a := application{
		Application: cli.Application{
			Name:    os.Args[0],
			Title:   "LUCI Deployment Tool",
			Context: func(context.Context) context.Context { return c },
			Commands: []*subcommands.Command{
				subcommands.CmdHelp,
				&cmdCheckout,
				&cmdDeploy,
				&cmdManage,

				authcli.SubcommandLogin(authOptions, "auth-login", false),
				authcli.SubcommandLogout(authOptions, "auth-logout", false),
				authcli.SubcommandInfo(authOptions, "auth-info", false),
			},
		},
	}
	logFlags := log.Config{
		Level: log.Warning,
	}

	var fs flag.FlagSet
	logFlags.AddFlags(&fs)
	a.authFlags.Register(&fs, authOptions)
	a.addFlags(&fs)
	if err := fs.Parse(args); err != nil {
		log.WithError(err).Errorf(c, "Failed to parse flags.")
		return 1
	}
	c = logFlags.Set(c)

	// Install a signal handler to cancel everything in response to a Ctrl+C.
	var cancelFunc context.CancelFunc
	c, cancelFunc = context.WithCancel(c)
	signalC := make(chan os.Signal, 1)
	signal.Notify(signalC, os.Interrupt)
	go func(c context.Context) {
		signalled := false
		for range signalC {
			if !signalled {
				signalled = true

				// First '^C'; soft-terminate
				log.Infof(c, "Received Control-C (keyboard interrupt), shutting down. Interrupt again for immediate exit.")
				cancelFunc()
			} else {
				// Multiple '^C'; terminate immediately
				os.Exit(1)
			}
		}
	}(c)
	defer func() {
		signal.Stop(signalC)
		close(signalC)
	}()

	// Load our layout file.
	if err := a.layout.load(c, a.layoutPath); err != nil {
		logError(c, err, "Failed to load layout file from [%s]", a.layoutPath)
		return 1
	}

	// Run our subcommand (and parse subcommand flags).
	return subcommands.Run(&a, fs.Args())
}

func main() {
	os.Exit(mainImpl(os.Args[1:]))
}
<commit_msg>Remove unnecessary usage of 'auth' and 'authcli' from luci_deploy.<commit_after>// Copyright 2016 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package main

import (
	"flag"
	"os"
	"os/signal"

	"github.com/maruel/subcommands"
	"golang.org/x/net/context"

	"github.com/luci/luci-go/common/cli"
	log "github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/logging/gologger"
)

type application struct {
	cli.Application

	layoutPath string
	workers    int

	layout deployLayout
	tools  tools
}

func (a *application) addFlags(fs *flag.FlagSet) {
	fs.StringVar(&a.layoutPath, "layout", a.layoutPath,
		"Path to the base layout file.")
	fs.IntVar(&a.workers, "workers", 0,
		"Maximum number of workers to use. If <= 0, no limit will be applied.")
}

func (a *application) runWork(c context.Context, fn func(*work) error) error {
	return runWork(c, a.workers, &a.tools, fn)
}

func mainImpl(args []string) int {
	c := gologger.StdConfig.Use(context.Background())
	a := application{
		Application: cli.Application{
			Name:    os.Args[0],
			Title:   "LUCI Deployment Tool",
			Context: func(context.Context) context.Context { return c },
			Commands: []*subcommands.Command{
				subcommands.CmdHelp,
				&cmdCheckout,
				&cmdDeploy,
				&cmdManage,
			},
		},
	}
	logFlags := log.Config{
		Level: log.Warning,
	}

	var fs flag.FlagSet
	logFlags.AddFlags(&fs)
	a.addFlags(&fs)
	if err := fs.Parse(args); err != nil {
		log.WithError(err).Errorf(c, "Failed to parse flags.")
		return 1
	}
	c = logFlags.Set(c)

	// Install a signal handler to cancel everything in response to a Ctrl+C.
	var cancelFunc context.CancelFunc
	c, cancelFunc = context.WithCancel(c)
	signalC := make(chan os.Signal, 1)
	signal.Notify(signalC, os.Interrupt)
	go func(c context.Context) {
		signalled := false
		for range signalC {
			if !signalled {
				signalled = true

				// First '^C'; soft-terminate
				log.Infof(c, "Received Control-C (keyboard interrupt), shutting down. Interrupt again for immediate exit.")
				cancelFunc()
			} else {
				// Multiple '^C'; terminate immediately
				os.Exit(1)
			}
		}
	}(c)
	defer func() {
		signal.Stop(signalC)
		close(signalC)
	}()

	// Load our layout file.
	if err := a.layout.load(c, a.layoutPath); err != nil {
		logError(c, err, "Failed to load layout file from [%s]", a.layoutPath)
		return 1
	}

	// Run our subcommand (and parse subcommand flags).
	return subcommands.Run(&a, fs.Args())
}

func main() {
	os.Exit(mainImpl(os.Args[1:]))
}
<|endoftext|>


<commit_before><commit_msg>moves.FixUp gets better HelpTExt and Name defaults. Part of #563.<commit_after><|endoftext|>


<commit_before><commit_msg>use insecure nessus and verbose mode in tests<commit_after><|endoftext|>


<commit_before>package uptime

import (
	"github.com/mozilla-services/heka/pipeline"
)

type UptimeFilter struct {
	// epoch
	startHour uint32
	endHour   uint32
	// in seconds
	initialUptime uint32
	lastUptime    uint32
	totalUptime   uint32
	// true if uptime dropped
	dived  bool
	output string
}

func (f *UptimeFilter) Init(config interface{}) error {
	return nil
}

func (f *UptimeFilter) Run(runner pipeline.FilterRunner, helper pipeline.PluginHelper) (err error) {
	var (
		pack    *pipeline.PipelinePack
		payload string
	)

	inChan := runner.InChan()
	for pack = range inChan {
		payload = pack.Message.GetPayload()
	}
	runner.LogMessage(payload)
	pack.Recycle()
	return
}

func init() {
	pipeline.RegisterPlugin("UptimeFilter", func() interface{} {
		return new(UptimeFilter)
	})
}
<commit_msg>process payload inside for loop<commit_after>package uptime

import (
	"github.com/mozilla-services/heka/pipeline"
)

type UptimeFilter struct {
	// epoch
	startHour uint32
	endHour   uint32
	// in seconds
	initialUptime uint32
	lastUptime    uint32
	totalUptime   uint32
	// true if uptime dropped
	dived  bool
	output string
}

func (f *UptimeFilter) Init(config interface{}) error {
	return nil
}

func (f *UptimeFilter) Run(runner pipeline.FilterRunner, helper pipeline.PluginHelper) (err error) {
	var (
		pack    *pipeline.PipelinePack
		payload string
	)

	inChan := runner.InChan()
	for pack = range inChan {
		payload = pack.Message.GetPayload()
		runner.LogMessage(payload)
		pack.Recycle()
	}
	return
}

func init() {
	pipeline.RegisterPlugin("UptimeFilter", func() interface{} {
		return new(UptimeFilter)
	})
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package etl

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"go.opencensus.io/trace"
	"golang.org/x/discovery/internal/config"
	"golang.org/x/discovery/internal/derrors"
	"golang.org/x/discovery/internal/fetch"
	"golang.org/x/discovery/internal/log"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/discovery/internal/proxy"
)

// fetchTimeout bounds the time allowed for fetching a single module.  It is
// mutable for testing purposes.
var fetchTimeout = 2 * config.StatementTimeout

const (
	// Indicates that although we have a valid module, some packages could not be processed.
	hasIncompletePackagesCode = 290
	hasIncompletePackagesDesc = "incomplete packages"
)

// ProxyRemoved is a set of module@version that have been removed from the proxy,
// even though they are still in the index.
var ProxyRemoved = map[string]bool{}

// fetchAndInsertVersion fetches the given module version from the module proxy
// or (in the case of the standard library) from the Go repo and writes the
// resulting data to the database.
//
// The given parentCtx is used for tracing, but fetches actually execute in a
// detached context with fixed timeout, so that fetches are allowed to complete
// even for short-lived requests.
func fetchAndInsertVersion(parentCtx context.Context, modulePath, version string, proxyClient *proxy.Client, db *postgres.DB) (hasIncompletePackages bool, goModPath string, err error) {
	defer derrors.Wrap(&err, "FetchAndInsertVersion(%q, %q)", modulePath, version)

	if ProxyRemoved[modulePath+"@"+version] {
		log.Infof(parentCtx, "not fetching %s@%s because it is on the ProxyRemoved list", modulePath, version)
		return false, "", derrors.Excluded
	}

	exc, err := db.IsExcluded(parentCtx, modulePath)
	if err != nil {
		return false, "", err
	}
	if exc {
		return false, "", derrors.Excluded
	}

	parentSpan := trace.FromContext(parentCtx)
	// A fixed timeout for FetchAndInsertVersion to allow module processing to
	// succeed even for extremely short lived requests.
	ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout)
	defer cancel()

	ctx, span := trace.StartSpanWithRemoteParent(ctx, "FetchAndInsertVersion", parentSpan.SpanContext())
	defer span.End()

	res, err := fetch.FetchVersion(ctx, modulePath, version, proxyClient)
	if res != nil {
		goModPath = res.GoModPath
	}
	if err != nil {
		return false, goModPath, err
	}
	log.Infof(ctx, "Fetched %s@%s", res.Version.ModulePath, res.Version.Version)
	if err = db.InsertVersion(ctx, res.Version); err != nil {
		return false, goModPath, err
	}
	log.Infof(ctx, "Inserted %s@%s", res.Version.ModulePath, res.Version.Version)
	return res.HasIncompletePackages, goModPath, nil
}

// fetchAndUpdateState fetches and processes a module version, and then updates
// the module_version_states table according to the result. It returns an HTTP
// status code representing the result of the fetch operation, and a non-nil
// error if this status code is not 200.
func fetchAndUpdateState(ctx context.Context, modulePath, version string, client *proxy.Client, db *postgres.DB) (_ int, err error) {
	defer derrors.Wrap(&err, "fetchAndUpdateState(%q, %q)", modulePath, version)

	ctx, span := trace.StartSpan(ctx, "fetchAndUpdateState")
	span.AddAttributes(
		trace.StringAttribute("modulePath", modulePath),
		trace.StringAttribute("version", version))
	defer span.End()
	var (
		code     = http.StatusOK
		fetchErr error
	)
	hasIncompletePackages, goModPath, fetchErr := fetchAndInsertVersion(ctx, modulePath, version, client, db)
	if fetchErr != nil {
		code = derrors.ToHTTPStatus(fetchErr)
		logf := log.Errorf
		if code < 500 {
			logf = log.Infof
		}
		logf(ctx, "Error executing fetch: %v (code %d)", fetchErr, code)
	}
	// TODO(b/147348928): return this information as an error from fetchAndInsertVersion.
	if hasIncompletePackages {
		code = hasIncompletePackagesCode
	}

	if code == http.StatusNotFound || code == http.StatusGone {
		log.Infof(ctx, "%s@%s: proxy said 404/410, deleting", modulePath, version)
		if err := db.DeleteVersion(ctx, nil, modulePath, version); err != nil {
			log.Error(ctx, err)
			return http.StatusInternalServerError, err
		}
	}

	// Update the module_version_states table with the new status of
	// module@version. This must happen last, because if it succeeds with a
	// code < 500 but a later action fails, we will never retry the later action.

	// TODO(b/139178863): Split UpsertModuleVersionState into InsertModuleVersionState and UpdateModuleVersionState.
	if err := db.UpsertModuleVersionState(ctx, modulePath, version, config.AppVersionLabel(), time.Time{}, code, goModPath, fetchErr); err != nil {
		log.Error(ctx, err)
		if fetchErr != nil {
			err = fmt.Errorf("error updating module version state: %v, original error: %v", err, fetchErr)
		}
		return http.StatusInternalServerError, err
	}
	log.Infof(ctx, "Updated module version state for %s@%s: code=%d, hasIncompletePackages=%t err=%v",
		modulePath, version, code, hasIncompletePackages, fetchErr)
	return code, fetchErr
}
<commit_msg>internal/etl: delete module on any fetch error<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package etl

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"go.opencensus.io/trace"
	"golang.org/x/discovery/internal/config"
	"golang.org/x/discovery/internal/derrors"
	"golang.org/x/discovery/internal/fetch"
	"golang.org/x/discovery/internal/log"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/discovery/internal/proxy"
)

// fetchTimeout bounds the time allowed for fetching a single module.  It is
// mutable for testing purposes.
var fetchTimeout = 2 * config.StatementTimeout

const (
	// Indicates that although we have a valid module, some packages could not be processed.
	hasIncompletePackagesCode = 290
	hasIncompletePackagesDesc = "incomplete packages"
)

// ProxyRemoved is a set of module@version that have been removed from the proxy,
// even though they are still in the index.
var ProxyRemoved = map[string]bool{}

// fetchAndInsertVersion fetches the given module version from the module proxy
// or (in the case of the standard library) from the Go repo and writes the
// resulting data to the database.
//
// The given parentCtx is used for tracing, but fetches actually execute in a
// detached context with fixed timeout, so that fetches are allowed to complete
// even for short-lived requests.
func fetchAndInsertVersion(parentCtx context.Context, modulePath, version string, proxyClient *proxy.Client, db *postgres.DB) (hasIncompletePackages bool, goModPath string, err error) {
	defer derrors.Wrap(&err, "FetchAndInsertVersion(%q, %q)", modulePath, version)

	if ProxyRemoved[modulePath+"@"+version] {
		log.Infof(parentCtx, "not fetching %s@%s because it is on the ProxyRemoved list", modulePath, version)
		return false, "", derrors.Excluded
	}

	exc, err := db.IsExcluded(parentCtx, modulePath)
	if err != nil {
		return false, "", err
	}
	if exc {
		return false, "", derrors.Excluded
	}

	parentSpan := trace.FromContext(parentCtx)
	// A fixed timeout for FetchAndInsertVersion to allow module processing to
	// succeed even for extremely short lived requests.
	ctx, cancel := context.WithTimeout(context.Background(), fetchTimeout)
	defer cancel()

	ctx, span := trace.StartSpanWithRemoteParent(ctx, "FetchAndInsertVersion", parentSpan.SpanContext())
	defer span.End()

	res, err := fetch.FetchVersion(ctx, modulePath, version, proxyClient)
	if res != nil {
		goModPath = res.GoModPath
	}
	if err != nil {
		return false, goModPath, err
	}
	log.Infof(ctx, "Fetched %s@%s", res.Version.ModulePath, res.Version.Version)
	if err = db.InsertVersion(ctx, res.Version); err != nil {
		return false, goModPath, err
	}
	log.Infof(ctx, "Inserted %s@%s", res.Version.ModulePath, res.Version.Version)
	return res.HasIncompletePackages, goModPath, nil
}

// fetchAndUpdateState fetches and processes a module version, and then updates
// the module_version_states table according to the result. It returns an HTTP
// status code representing the result of the fetch operation, and a non-nil
// error if this status code is not 200.
func fetchAndUpdateState(ctx context.Context, modulePath, version string, client *proxy.Client, db *postgres.DB) (_ int, err error) {
	defer derrors.Wrap(&err, "fetchAndUpdateState(%q, %q)", modulePath, version)

	ctx, span := trace.StartSpan(ctx, "fetchAndUpdateState")
	span.AddAttributes(
		trace.StringAttribute("modulePath", modulePath),
		trace.StringAttribute("version", version))
	defer span.End()
	var (
		code     = http.StatusOK
		fetchErr error
	)
	hasIncompletePackages, goModPath, fetchErr := fetchAndInsertVersion(ctx, modulePath, version, client, db)
	if fetchErr != nil {
		code = derrors.ToHTTPStatus(fetchErr)
		logf := log.Errorf
		if code < 500 {
			logf = log.Infof
		}
		logf(ctx, "Error executing fetch: %v (code %d)", fetchErr, code)
	}
	// TODO(b/147348928): return this information as an error from fetchAndInsertVersion.
	if hasIncompletePackages {
		code = hasIncompletePackagesCode
	}

	// If there were any errors processing the module then we didn't insert it.
	// Delete it in case we are reprocessing an existing module.

	if code > 400 {
		log.Infof(ctx, "%s@%s: code=%d, deleting", modulePath, version, code)
		if err := db.DeleteVersion(ctx, nil, modulePath, version); err != nil {
			log.Error(ctx, err)
			return http.StatusInternalServerError, err
		}
	}

	// Update the module_version_states table with the new status of
	// module@version. This must happen last, because if it succeeds with a
	// code < 500 but a later action fails, we will never retry the later action.

	// TODO(b/139178863): Split UpsertModuleVersionState into InsertModuleVersionState and UpdateModuleVersionState.
	if err := db.UpsertModuleVersionState(ctx, modulePath, version, config.AppVersionLabel(), time.Time{}, code, goModPath, fetchErr); err != nil {
		log.Error(ctx, err)
		if fetchErr != nil {
			err = fmt.Errorf("error updating module version state: %v, original error: %v", err, fetchErr)
		}
		return http.StatusInternalServerError, err
	}
	log.Infof(ctx, "Updated module version state for %s@%s: code=%d, hasIncompletePackages=%t err=%v",
		modulePath, version, code, hasIncompletePackages, fetchErr)
	return code, fetchErr
}
<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package relui

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/fs"
	"log"
	"mime"
	"net/http"
	"net/url"
	"path"

	"github.com/google/uuid"
	"github.com/jackc/pgx/v4/pgxpool"
	"golang.org/x/build/internal/relui/db"
	"golang.org/x/build/internal/workflow"
)

// fileServerHandler returns a http.Handler rooted at root. It will
// call the next handler provided for requests to "/".
//
// The returned handler sets the appropriate Content-Type and
// Cache-Control headers for the returned file.
func fileServerHandler(fs fs.FS, next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/" {
			next.ServeHTTP(w, r)
			return
		}
		w.Header().Set("Content-Type", mime.TypeByExtension(path.Ext(r.URL.Path)))
		w.Header().Set("Cache-Control", "no-cache, private, max-age=0")
		s := http.FileServer(http.FS(fs))
		s.ServeHTTP(w, r)
	})
}

// Server implements the http handlers for relui.
type Server struct {
	db      *pgxpool.Pool
	m       *http.ServeMux
	w       *Worker
	baseURL *url.URL

	homeTmpl        *template.Template
	newWorkflowTmpl *template.Template
}

// NewServer initializes a server with the provided connection pool.
func NewServer(p *pgxpool.Pool, w *Worker, baseURL *url.URL) *Server {
	s := &Server{
		db:      p,
		m:       new(http.ServeMux),
		w:       w,
		baseURL: baseURL,
	}
	helpers := map[string]interface{}{
		"baseLink": s.BaseLink,
	}
	layout := template.Must(template.New("layout.html").Funcs(helpers).ParseFS(templates, "templates/layout.html"))
	s.homeTmpl = template.Must(template.Must(layout.Clone()).Funcs(helpers).ParseFS(templates, "templates/home.html"))
	s.newWorkflowTmpl = template.Must(template.Must(layout.Clone()).Funcs(helpers).ParseFS(templates, "templates/new_workflow.html"))
	s.m.Handle("/workflows/create", http.HandlerFunc(s.createWorkflowHandler))
	s.m.Handle("/workflows/new", http.HandlerFunc(s.newWorkflowHandler))
	s.m.Handle("/", fileServerHandler(static, http.HandlerFunc(s.homeHandler)))
	return s
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if s.baseURL == nil || s.baseURL.Path == "/" {
		s.m.ServeHTTP(w, r)
		return
	}
	http.StripPrefix(s.baseURL.Path, s.m)
}

func (s *Server) Serve(port string) error {
	return http.ListenAndServe(":"+port, s.m)
}

func (s *Server) BaseLink(target string) string {
	if s.baseURL == nil {
		return target
	}
	u, err := url.Parse(target)
	if err != nil {
		log.Printf("BaseLink: url.Parse(%q) = %v, %v", target, u, err)
		return target
	}
	if u.IsAbs() {
		return u.String()
	}
	u.Scheme = s.baseURL.Scheme
	u.Host = s.baseURL.Host
	u.Path = path.Join(s.baseURL.Path, u.Path)
	return u.String()
}

type homeResponse struct {
	Workflows     []db.Workflow
	WorkflowTasks map[uuid.UUID][]db.Task
	TaskLogs      map[uuid.UUID]map[string][]db.TaskLog
}

func (h *homeResponse) Logs(workflow uuid.UUID, task string) []db.TaskLog {
	t := h.TaskLogs[workflow]
	if t == nil {
		return nil
	}
	return t[task]
}

func (h *homeResponse) WorkflowParams(wf db.Workflow) map[string]string {
	params := make(map[string]string)
	json.Unmarshal([]byte(wf.Params.String), &params)
	return params
}

// homeHandler renders the homepage.
func (s *Server) homeHandler(w http.ResponseWriter, r *http.Request) {
	resp, err := s.buildHomeResponse(r.Context())
	if err != nil {
		log.Printf("homeHandler: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	out := bytes.Buffer{}
	if err := s.homeTmpl.Execute(&out, resp); err != nil {
		log.Printf("homeHandler: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	io.Copy(w, &out)
}

func (s *Server) buildHomeResponse(ctx context.Context) (*homeResponse, error) {
	q := db.New(s.db)
	ws, err := q.Workflows(ctx)
	if err != nil {
		return nil, err
	}
	tasks, err := q.Tasks(ctx)
	if err != nil {
		return nil, err
	}
	wfTasks := make(map[uuid.UUID][]db.Task, len(ws))
	for _, t := range tasks {
		wfTasks[t.WorkflowID] = append(wfTasks[t.WorkflowID], t)
	}
	tlogs, err := q.TaskLogs(ctx)
	if err != nil {
		return nil, err
	}
	wftlogs := make(map[uuid.UUID]map[string][]db.TaskLog)
	for _, l := range tlogs {
		if wftlogs[l.WorkflowID] == nil {
			wftlogs[l.WorkflowID] = make(map[string][]db.TaskLog)
		}
		wftlogs[l.WorkflowID][l.TaskName] = append(wftlogs[l.WorkflowID][l.TaskName], l)
	}
	return &homeResponse{Workflows: ws, WorkflowTasks: wfTasks, TaskLogs: wftlogs}, nil
}

type newWorkflowResponse struct {
	Definitions map[string]*workflow.Definition
	Name        string
}

func (n *newWorkflowResponse) Selected() *workflow.Definition {
	return n.Definitions[n.Name]
}

// newWorkflowHandler presents a form for creating a new workflow.
func (s *Server) newWorkflowHandler(w http.ResponseWriter, r *http.Request) {
	out := bytes.Buffer{}
	resp := &newWorkflowResponse{
		Definitions: Definitions(),
		Name:        r.FormValue("workflow.name"),
	}
	if err := s.newWorkflowTmpl.Execute(&out, resp); err != nil {
		log.Printf("newWorkflowHandler: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	io.Copy(w, &out)
}

// createWorkflowHandler persists a new workflow in the datastore, and
// starts the workflow in a goroutine.
func (s *Server) createWorkflowHandler(w http.ResponseWriter, r *http.Request) {
	name := r.FormValue("workflow.name")
	d := Definition(name)
	if d == nil {
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}
	params := make(map[string]string)
	for _, n := range d.ParameterNames() {
		params[n] = r.FormValue(fmt.Sprintf("workflow.params.%s", n))
		if params[n] == "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}
	}
	if _, err := s.w.StartWorkflow(r.Context(), name, d, params); err != nil {
		log.Printf("s.w.StartWorkflow(%v, %v, %v): %v", r.Context(), d, params, err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	http.Redirect(w, r, "/", http.StatusSeeOther)
}
<commit_msg>internal/relui: correct baseURL behavior<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package relui

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"html/template"
	"io"
	"io/fs"
	"log"
	"mime"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/google/uuid"
	"github.com/jackc/pgx/v4/pgxpool"
	"golang.org/x/build/internal/relui/db"
	"golang.org/x/build/internal/workflow"
)

// fileServerHandler returns a http.Handler rooted at root. It will
// call the next handler provided for requests to "/".
//
// The returned handler sets the appropriate Content-Type and
// Cache-Control headers for the returned file.
func fileServerHandler(fs fs.FS, next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/" {
			next.ServeHTTP(w, r)
			return
		}
		w.Header().Set("Content-Type", mime.TypeByExtension(path.Ext(r.URL.Path)))
		w.Header().Set("Cache-Control", "no-cache, private, max-age=0")
		s := http.FileServer(http.FS(fs))
		s.ServeHTTP(w, r)
	})
}

// Server implements the http handlers for relui.
type Server struct {
	db      *pgxpool.Pool
	m       *http.ServeMux
	w       *Worker
	baseURL *url.URL
	// mux used if baseURL is set
	bm *http.ServeMux

	homeTmpl        *template.Template
	newWorkflowTmpl *template.Template
}

// NewServer initializes a server with the provided connection pool.
func NewServer(p *pgxpool.Pool, w *Worker, baseURL *url.URL) *Server {
	s := &Server{
		db:      p,
		m:       new(http.ServeMux),
		w:       w,
		baseURL: baseURL,
	}
	helpers := map[string]interface{}{
		"baseLink": s.BaseLink,
	}
	layout := template.Must(template.New("layout.html").Funcs(helpers).ParseFS(templates, "templates/layout.html"))
	s.homeTmpl = template.Must(template.Must(layout.Clone()).Funcs(helpers).ParseFS(templates, "templates/home.html"))
	s.newWorkflowTmpl = template.Must(template.Must(layout.Clone()).Funcs(helpers).ParseFS(templates, "templates/new_workflow.html"))
	s.m.Handle("/workflows/create", http.HandlerFunc(s.createWorkflowHandler))
	s.m.Handle("/workflows/new", http.HandlerFunc(s.newWorkflowHandler))
	s.m.Handle("/", fileServerHandler(static, http.HandlerFunc(s.homeHandler)))
	if baseURL != nil && baseURL.Path != "/" && baseURL.Path != "" {
		nosuffix := strings.TrimSuffix(baseURL.Path, "/")
		s.bm = new(http.ServeMux)
		s.bm.Handle(nosuffix+"/", http.StripPrefix(nosuffix, s.m))
		s.bm.Handle("/", s.m)
	}
	return s
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if s.bm != nil {
		s.bm.ServeHTTP(w, r)
		return
	}
	s.m.ServeHTTP(w, r)
}

func (s *Server) BaseLink(target string) string {
	if s.baseURL == nil {
		return target
	}
	u, err := url.Parse(target)
	if err != nil {
		log.Printf("BaseLink: url.Parse(%q) = %v, %v", target, u, err)
		return target
	}
	if u.IsAbs() {
		return u.String()
	}
	u.Scheme = s.baseURL.Scheme
	u.Host = s.baseURL.Host
	u.Path = path.Join(s.baseURL.Path, u.Path)
	return u.String()
}

type homeResponse struct {
	Workflows     []db.Workflow
	WorkflowTasks map[uuid.UUID][]db.Task
	TaskLogs      map[uuid.UUID]map[string][]db.TaskLog
}

func (h *homeResponse) Logs(workflow uuid.UUID, task string) []db.TaskLog {
	t := h.TaskLogs[workflow]
	if t == nil {
		return nil
	}
	return t[task]
}

func (h *homeResponse) WorkflowParams(wf db.Workflow) map[string]string {
	params := make(map[string]string)
	json.Unmarshal([]byte(wf.Params.String), &params)
	return params
}

// homeHandler renders the homepage.
func (s *Server) homeHandler(w http.ResponseWriter, r *http.Request) {
	resp, err := s.buildHomeResponse(r.Context())
	if err != nil {
		log.Printf("homeHandler: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	out := bytes.Buffer{}
	if err := s.homeTmpl.Execute(&out, resp); err != nil {
		log.Printf("homeHandler: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	io.Copy(w, &out)
}

func (s *Server) buildHomeResponse(ctx context.Context) (*homeResponse, error) {
	q := db.New(s.db)
	ws, err := q.Workflows(ctx)
	if err != nil {
		return nil, err
	}
	tasks, err := q.Tasks(ctx)
	if err != nil {
		return nil, err
	}
	wfTasks := make(map[uuid.UUID][]db.Task, len(ws))
	for _, t := range tasks {
		wfTasks[t.WorkflowID] = append(wfTasks[t.WorkflowID], t)
	}
	tlogs, err := q.TaskLogs(ctx)
	if err != nil {
		return nil, err
	}
	wftlogs := make(map[uuid.UUID]map[string][]db.TaskLog)
	for _, l := range tlogs {
		if wftlogs[l.WorkflowID] == nil {
			wftlogs[l.WorkflowID] = make(map[string][]db.TaskLog)
		}
		wftlogs[l.WorkflowID][l.TaskName] = append(wftlogs[l.WorkflowID][l.TaskName], l)
	}
	return &homeResponse{Workflows: ws, WorkflowTasks: wfTasks, TaskLogs: wftlogs}, nil
}

type newWorkflowResponse struct {
	Definitions map[string]*workflow.Definition
	Name        string
}

func (n *newWorkflowResponse) Selected() *workflow.Definition {
	return n.Definitions[n.Name]
}

// newWorkflowHandler presents a form for creating a new workflow.
func (s *Server) newWorkflowHandler(w http.ResponseWriter, r *http.Request) {
	out := bytes.Buffer{}
	resp := &newWorkflowResponse{
		Definitions: Definitions(),
		Name:        r.FormValue("workflow.name"),
	}
	if err := s.newWorkflowTmpl.Execute(&out, resp); err != nil {
		log.Printf("newWorkflowHandler: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	io.Copy(w, &out)
}

// createWorkflowHandler persists a new workflow in the datastore, and
// starts the workflow in a goroutine.
func (s *Server) createWorkflowHandler(w http.ResponseWriter, r *http.Request) {
	name := r.FormValue("workflow.name")
	d := Definition(name)
	if d == nil {
		http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
		return
	}
	params := make(map[string]string)
	for _, n := range d.ParameterNames() {
		params[n] = r.FormValue(fmt.Sprintf("workflow.params.%s", n))
		if params[n] == "" {
			http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest)
			return
		}
	}
	if _, err := s.w.StartWorkflow(r.Context(), name, d, params); err != nil {
		log.Printf("s.w.StartWorkflow(%v, %v, %v): %v", r.Context(), d, params, err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	http.Redirect(w, r, "/", http.StatusSeeOther)
}
<|endoftext|>


<commit_before>package gokiq

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"math/rand"
	"os"
	"os/signal"
	"reflect"
	"sync"
	"syscall"
	"time"

	"github.com/garyburd/redigo/redis"
)

type Job struct {
	Type  string        `json:"class"`
	Args  []interface{} `json:"args"`
	Queue string        `json:"queue,omitempty"`
	ID    string        `json:"jid"`

	Retry interface{} `json:"retry"` // can be int (number of retries) or bool (true means default)

	MaxRetries   int    `json:"-"`
	RetryCount   int    `json:"retry_count"`
	ErrorMessage string `json:"error_message,omitempty"`
	ErrorType    string `json:"error_class,omitempty"`
	RetriedAt    string `json:"retried_at,omitempty"`
	FailedAt     string `json:"failed_at,omitempty"`

	StartTime time.Time `json:"-"`
}

func (job *Job) FromJSON(data []byte) error {
	err := json.Unmarshal(data, job)
	if err != nil {
		return err
	}
	if max, ok := job.Retry.(float64); ok {
		job.MaxRetries = int(max)
	} else if r, ok := job.Retry.(bool); ok && !r {
	} else {
		job.MaxRetries = defaultMaxRetries
	}
	return nil
}

func (job *Job) JSON() []byte {
	res, _ := json.Marshal(job)
	return res
}

type message struct {
	job *Job
	die bool
}

const (
	TimestampFormat     = "2006-01-02 15:04:05 MST"
	redisTimeout        = 1
	defaultMaxRetries   = 25
	defaultPollInterval = 5
	defaultWorkerCount  = 25
	defaultRedisServer  = "127.0.0.1:6379"
	keyExpiry           = 86400 // one day
)

type QueueConfig map[string]int

func (q QueueConfig) String() string {
	str := ""
	for queue, priority := range q {
		str += fmt.Sprintf("%s=%d,", queue, priority)
	}
	return str[:len(str)-1]
}

type Worker interface {
	Perform([]interface{}) error
}

var Workers = NewWorkerConfig()

type WorkerConfig struct {
	RedisServer    string // TODO: allow specifying redis db
	RedisNamespace string
	Queues         QueueConfig
	WorkerCount    int
	PollInterval   int
	ReportError    func(error, *Job) // TODO: pass in a stack trace for context

	workerMapping map[string]reflect.Type
	randomQueues  []string
	redisPool     *redis.Pool
	workQueue     chan message
	doneQueue     chan bool
	sync.RWMutex  // R is locked by Run() and scheduler(), W is locked by quitHandler() when it receives a signal
}

func NewWorkerConfig() *WorkerConfig {
	return &WorkerConfig{
		RedisServer:   defaultRedisServer,
		PollInterval:  defaultPollInterval,
		WorkerCount:   defaultWorkerCount,
		Queues:        QueueConfig{"default": 1},
		ReportError:   func(error, *Job) {},
		workerMapping: make(map[string]reflect.Type),
		workQueue:     make(chan message),
		doneQueue:     make(chan bool),
	}
}

func (w *WorkerConfig) Register(name string, worker Worker) {
	w.workerMapping[name] = workerType(worker)
}

func (w *WorkerConfig) Run() {
	log.Printf(`state=starting worker_count=%d redis=%s/0/%s queues="%s" pid=%d`, w.WorkerCount, w.RedisServer, w.RedisNamespace, w.Queues, pid)
	w.denormalizeQueues()
	w.connectRedis()

	for i := 0; i < w.WorkerCount; i++ {
		go w.worker(workerID(i))
	}

	go w.scheduler()
	go w.quitHandler()

	log.Printf(`state=started pid=%d`, pid)
	for {
		w.RLock() // don't let quitHandler() stop us in the middle of a job
		msg, err := redis.Values(w.redisQuery("BLPOP", append(w.queueList(), redisTimeout)...))
		if err == redis.ErrNil {
			continue
		}
		if err != nil {
			w.handleError(err)
			time.Sleep(redisTimeout * time.Second) // likely a transient redis error, sleep before retrying
			continue
		}

		job := &Job{}
		err = job.FromJSON(msg[1].([]byte))
		if err != nil {
			w.handleError(err)
			continue
		}
		job.Queue = string(msg[0].([]byte)[len(w.nsKey("queue:")):])
		w.workQueue <- message{job: job}

		w.RUnlock()
	}
}

// create a slice of queues with duplicates using the assigned frequencies
func (w *WorkerConfig) denormalizeQueues() {
	for queue, x := range w.Queues {
		for i := 0; i < x; i++ {
			w.randomQueues = append(w.randomQueues, w.nsKey("queue:"+queue))
		}
	}
}

// get a random slice of unique queues from the slice of denormalized queues
func (w *WorkerConfig) queueList() []interface{} {
	size := len(w.Queues)
	res := make([]interface{}, 0, size)
	queues := make(map[string]bool, size)

	indices := rand.Perm(len(w.randomQueues))[:size]
	for _, i := range indices {
		queue := w.randomQueues[i]
		if _, ok := queues[queue]; !ok {
			queues[queue] = true
			res = append(res, queue)
		}
	}

	return res
}

func (w *WorkerConfig) handleError(err error) {
	log.Printf(`event=error error_type=%T error_message="%s" pid=%d`, err, err, pid)
	w.ReportError(err, nil)
}

// checks the sorted set of scheduled jobs and retries and queues them when it's time
// TODO: move this to a Lua script
func (w *WorkerConfig) scheduler() {
	pollSets := []string{w.nsKey("retry"), w.nsKey("schedule")}

	for _ = range time.Tick(time.Duration(w.PollInterval) * time.Second) {
		w.RLock() // don't let quitHandler() stop us in the middle of a run
		conn := w.redisPool.Get()
		now := fmt.Sprintf("%f", currentTimeFloat())
		for _, set := range pollSets {
			conn.Send("MULTI")
			conn.Send("ZRANGEBYSCORE", set, "-inf", now)
			conn.Send("ZREMRANGEBYSCORE", set, "-inf", now)
			res, err := redis.Values(conn.Do("EXEC"))
			if err != nil {
				w.handleError(err)
				continue
			}

			for _, msg := range res[0].([]interface{}) {
				parsedMsg := &struct {
					Queue string `json:"queue"`
				}{}
				msgBytes := msg.([]byte)
				err := json.Unmarshal(msgBytes, parsedMsg)
				if err != nil {
					w.handleError(err)
					continue
				}
				conn.Send("MULTI")
				conn.Send("SADD", w.nsKey("queues"), parsedMsg.Queue)
				conn.Send("RPUSH", w.nsKey("queue:"+parsedMsg.Queue), msgBytes)
				_, err = conn.Do("EXEC")
				if err != nil {
					w.handleError(err)
				}
			}
		}
		conn.Close()
		w.RUnlock()
	}
}

// listens for SIGINT, SIGTERM, and SIGQUIT to perform a clean shutdown
func (w *WorkerConfig) quitHandler() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	signal.Notify(c, syscall.SIGQUIT)

	for sig := range c {
		log.Printf("state=stopping signal=%s pid=%d", sig, pid)
		w.Lock()           // wait for the current run loop and scheduler iterations to finish
		close(w.workQueue) // tell worker goroutines to stop after they finish their current job
		for i := 0; i < w.WorkerCount; i++ {
			<-w.doneQueue // wait for workers to finish
		}
		log.Printf("state=stopped pid=%d", pid)
		os.Exit(0)
	}
}

func (w *WorkerConfig) connectRedis() {
	// TODO: add a mutex for the redis pool
	if w.redisPool != nil {
		w.redisPool.Close()
	}
	w.redisPool = redis.NewPool(func() (redis.Conn, error) {
		return redis.Dial("tcp", w.RedisServer)
	}, w.WorkerCount+1)
}

func (w *WorkerConfig) redisQuery(command string, args ...interface{}) (interface{}, error) {
	conn := w.redisPool.Get()
	defer conn.Close()
	return conn.Do(command, args...)
}

func (w *WorkerConfig) worker(id string) {
	for msg := range w.workQueue {
		if msg.die {
			return
		}

		job := msg.job
		typ, ok := w.workerMapping[msg.job.Type]
		if !ok {
			err := fmt.Errorf("Unknown worker type: %s", job.Type)
			w.scheduleRetry(job, err)
			continue
		}

		w.logJobStart(job, id)

		// wrap Perform() in a function so that we can recover from panics
		var err error
		func() {
			defer func() {
				if r := recover(); r != nil {
					// TODO: log stack trace
					err = panicToError(r)
				}
			}()
			err = reflect.New(typ).Interface().(Worker).Perform(msg.job.Args)
		}()
		if err != nil {
			w.scheduleRetry(job, err)
		}
		w.logJobFinish(job, id, err == nil)
	}
	w.doneQueue <- true
}

func (w *WorkerConfig) scheduleRetry(job *Job, err error) {
	w.ReportError(err, job)

	now := time.Now().UTC().Format(TimestampFormat)
	if job.FailedAt == "" {
		job.FailedAt = now
	} else {
		job.RetryCount += 1
	}
	if job.RetryCount > 0 {
		job.RetriedAt = now
	}

	log.Printf(`event=job_error job_id=%s job_type=%s queue=%s retries=%d max_retries=%d error_type=%T error_message="%s" pid=%d`, job.ID, job.Type, job.Queue, job.RetryCount, job.MaxRetries, err, err, pid)

	if job.RetryCount < job.MaxRetries {
		job.ErrorType = fmt.Sprintf("%T", err)
		job.ErrorMessage = err.Error()

		nextRetry := currentTimeFloat() + retryDelay(job.RetryCount)

		w.redisQuery("ZADD", w.nsKey("retry"), fmt.Sprintf("%f", nextRetry), job.JSON())
	}
}

type runningJob struct {
	Queue     string `json:"queue"`
	Job       *Job   `json:"payload"`
	Timestamp int64  `json:"run_at"`
}

// TODO: make a lua script for this
func (w *WorkerConfig) logJobStart(job *Job, workerID string) {
	conn := w.redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SADD", w.nsKey("workers"), workerID)
	conn.Send("SETEX", w.nsKey("worker:"+workerID+":started"), keyExpiry, time.Now().UTC().String())
	payload := &runningJob{job.Queue, job, time.Now().Unix()}
	json, _ := json.Marshal(payload)
	conn.Send("SETEX", w.nsKey("worker:"+workerID), keyExpiry, json)
	_, err := conn.Do("EXEC")
	if err != nil {
		w.handleError(err)
	}

	job.StartTime = time.Now()
	log.Printf("event=job_start job_id=%s job_type=%s queue=%s worker_id=%s pid=%d", job.ID, job.Type, job.Queue, workerID, pid)
}

// TODO: make a lua script for this
func (w *WorkerConfig) logJobFinish(job *Job, workerID string, success bool) {
	log.Printf("event=job_finish job_id=%s job_type=%s queue=%s duration=%v success=%t worker_id=%s pid=%d", job.ID, job.Type, job.Queue, time.Since(job.StartTime), success, workerID, pid)

	conn := w.redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SREM", w.nsKey("workers"), workerID)
	conn.Send("DEL", w.nsKey("worker:"+workerID+":started"))
	conn.Send("DEL", w.nsKey("worker:"+workerID))
	conn.Send("INCR", w.nsKey("stat:processed"))
	if !success {
		conn.Send("INCR", w.nsKey("stat:failed"))
	}
	_, err := conn.Do("EXEC")
	if err != nil {
		w.handleError(err)
	}
}

func (w *WorkerConfig) nsKey(key string) string {
	if w.RedisNamespace != "" {
		return w.RedisNamespace + ":" + key
	}
	return key
}

// formula from Sidekiq (originally from delayed_job)
func retryDelay(count int) float64 {
	return math.Pow(float64(count), 4) + 15 + (float64(rand.Intn(30)) * (float64(count) + 1))
}

func currentTimeFloat() float64 {
	return float64(time.Now().UnixNano()) / float64(time.Second)
}

func panicToError(err interface{}) error {
	if str, ok := err.(string); ok {
		return fmt.Errorf(str)
	}
	return err.(error)
}

var (
	pid         = os.Getpid()
	hostname, _ = os.Hostname()
)

func workerID(i int) string {
	return fmt.Sprintf("%s:%d-%d", hostname, pid, i)
}

func workerType(worker Worker) reflect.Type {
	return reflect.Indirect(reflect.ValueOf(worker)).Type()
}
<commit_msg>Make unknown worker error better<commit_after>package gokiq

import (
	"encoding/json"
	"fmt"
	"log"
	"math"
	"math/rand"
	"os"
	"os/signal"
	"reflect"
	"sync"
	"syscall"
	"time"

	"github.com/garyburd/redigo/redis"
)

type Job struct {
	Type  string        `json:"class"`
	Args  []interface{} `json:"args"`
	Queue string        `json:"queue,omitempty"`
	ID    string        `json:"jid"`

	Retry interface{} `json:"retry"` // can be int (number of retries) or bool (true means default)

	MaxRetries   int    `json:"-"`
	RetryCount   int    `json:"retry_count"`
	ErrorMessage string `json:"error_message,omitempty"`
	ErrorType    string `json:"error_class,omitempty"`
	RetriedAt    string `json:"retried_at,omitempty"`
	FailedAt     string `json:"failed_at,omitempty"`

	StartTime time.Time `json:"-"`
}

func (job *Job) FromJSON(data []byte) error {
	err := json.Unmarshal(data, job)
	if err != nil {
		return err
	}
	if max, ok := job.Retry.(float64); ok {
		job.MaxRetries = int(max)
	} else if r, ok := job.Retry.(bool); ok && !r {
	} else {
		job.MaxRetries = defaultMaxRetries
	}
	return nil
}

func (job *Job) JSON() []byte {
	res, _ := json.Marshal(job)
	return res
}

type message struct {
	job *Job
	die bool
}

const (
	TimestampFormat     = "2006-01-02 15:04:05 MST"
	redisTimeout        = 1
	defaultMaxRetries   = 25
	defaultPollInterval = 5
	defaultWorkerCount  = 25
	defaultRedisServer  = "127.0.0.1:6379"
	keyExpiry           = 86400 // one day
)

type QueueConfig map[string]int

func (q QueueConfig) String() string {
	str := ""
	for queue, priority := range q {
		str += fmt.Sprintf("%s=%d,", queue, priority)
	}
	return str[:len(str)-1]
}

type Worker interface {
	Perform([]interface{}) error
}

var Workers = NewWorkerConfig()

type WorkerConfig struct {
	RedisServer    string // TODO: allow specifying redis db
	RedisNamespace string
	Queues         QueueConfig
	WorkerCount    int
	PollInterval   int
	ReportError    func(error, *Job) // TODO: pass in a stack trace for context

	workerMapping map[string]reflect.Type
	randomQueues  []string
	redisPool     *redis.Pool
	workQueue     chan message
	doneQueue     chan bool
	sync.RWMutex  // R is locked by Run() and scheduler(), W is locked by quitHandler() when it receives a signal
}

func NewWorkerConfig() *WorkerConfig {
	return &WorkerConfig{
		RedisServer:   defaultRedisServer,
		PollInterval:  defaultPollInterval,
		WorkerCount:   defaultWorkerCount,
		Queues:        QueueConfig{"default": 1},
		ReportError:   func(error, *Job) {},
		workerMapping: make(map[string]reflect.Type),
		workQueue:     make(chan message),
		doneQueue:     make(chan bool),
	}
}

func (w *WorkerConfig) Register(name string, worker Worker) {
	w.workerMapping[name] = workerType(worker)
}

func (w *WorkerConfig) Run() {
	log.Printf(`state=starting worker_count=%d redis=%s/0/%s queues="%s" pid=%d`, w.WorkerCount, w.RedisServer, w.RedisNamespace, w.Queues, pid)
	w.denormalizeQueues()
	w.connectRedis()

	for i := 0; i < w.WorkerCount; i++ {
		go w.worker(workerID(i))
	}

	go w.scheduler()
	go w.quitHandler()

	log.Printf(`state=started pid=%d`, pid)
	for {
		w.RLock() // don't let quitHandler() stop us in the middle of a job
		msg, err := redis.Values(w.redisQuery("BLPOP", append(w.queueList(), redisTimeout)...))
		if err == redis.ErrNil {
			continue
		}
		if err != nil {
			w.handleError(err)
			time.Sleep(redisTimeout * time.Second) // likely a transient redis error, sleep before retrying
			continue
		}

		job := &Job{}
		err = job.FromJSON(msg[1].([]byte))
		if err != nil {
			w.handleError(err)
			continue
		}
		job.Queue = string(msg[0].([]byte)[len(w.nsKey("queue:")):])
		w.workQueue <- message{job: job}

		w.RUnlock()
	}
}

// create a slice of queues with duplicates using the assigned frequencies
func (w *WorkerConfig) denormalizeQueues() {
	for queue, x := range w.Queues {
		for i := 0; i < x; i++ {
			w.randomQueues = append(w.randomQueues, w.nsKey("queue:"+queue))
		}
	}
}

// get a random slice of unique queues from the slice of denormalized queues
func (w *WorkerConfig) queueList() []interface{} {
	size := len(w.Queues)
	res := make([]interface{}, 0, size)
	queues := make(map[string]bool, size)

	indices := rand.Perm(len(w.randomQueues))[:size]
	for _, i := range indices {
		queue := w.randomQueues[i]
		if _, ok := queues[queue]; !ok {
			queues[queue] = true
			res = append(res, queue)
		}
	}

	return res
}

func (w *WorkerConfig) handleError(err error) {
	log.Printf(`event=error error_type=%T error_message="%s" pid=%d`, err, err, pid)
	w.ReportError(err, nil)
}

// checks the sorted set of scheduled jobs and retries and queues them when it's time
// TODO: move this to a Lua script
func (w *WorkerConfig) scheduler() {
	pollSets := []string{w.nsKey("retry"), w.nsKey("schedule")}

	for _ = range time.Tick(time.Duration(w.PollInterval) * time.Second) {
		w.RLock() // don't let quitHandler() stop us in the middle of a run
		conn := w.redisPool.Get()
		now := fmt.Sprintf("%f", currentTimeFloat())
		for _, set := range pollSets {
			conn.Send("MULTI")
			conn.Send("ZRANGEBYSCORE", set, "-inf", now)
			conn.Send("ZREMRANGEBYSCORE", set, "-inf", now)
			res, err := redis.Values(conn.Do("EXEC"))
			if err != nil {
				w.handleError(err)
				continue
			}

			for _, msg := range res[0].([]interface{}) {
				parsedMsg := &struct {
					Queue string `json:"queue"`
				}{}
				msgBytes := msg.([]byte)
				err := json.Unmarshal(msgBytes, parsedMsg)
				if err != nil {
					w.handleError(err)
					continue
				}
				conn.Send("MULTI")
				conn.Send("SADD", w.nsKey("queues"), parsedMsg.Queue)
				conn.Send("RPUSH", w.nsKey("queue:"+parsedMsg.Queue), msgBytes)
				_, err = conn.Do("EXEC")
				if err != nil {
					w.handleError(err)
				}
			}
		}
		conn.Close()
		w.RUnlock()
	}
}

// listens for SIGINT, SIGTERM, and SIGQUIT to perform a clean shutdown
func (w *WorkerConfig) quitHandler() {
	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	signal.Notify(c, syscall.SIGQUIT)

	for sig := range c {
		log.Printf("state=stopping signal=%s pid=%d", sig, pid)
		w.Lock()           // wait for the current run loop and scheduler iterations to finish
		close(w.workQueue) // tell worker goroutines to stop after they finish their current job
		for i := 0; i < w.WorkerCount; i++ {
			<-w.doneQueue // wait for workers to finish
		}
		log.Printf("state=stopped pid=%d", pid)
		os.Exit(0)
	}
}

func (w *WorkerConfig) connectRedis() {
	// TODO: add a mutex for the redis pool
	if w.redisPool != nil {
		w.redisPool.Close()
	}
	w.redisPool = redis.NewPool(func() (redis.Conn, error) {
		return redis.Dial("tcp", w.RedisServer)
	}, w.WorkerCount+1)
}

func (w *WorkerConfig) redisQuery(command string, args ...interface{}) (interface{}, error) {
	conn := w.redisPool.Get()
	defer conn.Close()
	return conn.Do(command, args...)
}

func (w *WorkerConfig) worker(id string) {
	for msg := range w.workQueue {
		if msg.die {
			return
		}

		job := msg.job
		typ, ok := w.workerMapping[msg.job.Type]
		if !ok {
			err := UnknownWorkerError{job.Type}
			w.scheduleRetry(job, err)
			continue
		}

		w.logJobStart(job, id)

		// wrap Perform() in a function so that we can recover from panics
		var err error
		func() {
			defer func() {
				if r := recover(); r != nil {
					// TODO: log stack trace
					err = panicToError(r)
				}
			}()
			err = reflect.New(typ).Interface().(Worker).Perform(msg.job.Args)
		}()
		if err != nil {
			w.scheduleRetry(job, err)
		}
		w.logJobFinish(job, id, err == nil)
	}
	w.doneQueue <- true
}

func (w *WorkerConfig) scheduleRetry(job *Job, err error) {
	w.ReportError(err, job)

	now := time.Now().UTC().Format(TimestampFormat)
	if job.FailedAt == "" {
		job.FailedAt = now
	} else {
		job.RetryCount += 1
	}
	if job.RetryCount > 0 {
		job.RetriedAt = now
	}

	log.Printf(`event=job_error job_id=%s job_type=%s queue=%s retries=%d max_retries=%d error_type=%T error_message="%s" pid=%d`, job.ID, job.Type, job.Queue, job.RetryCount, job.MaxRetries, err, err, pid)

	if job.RetryCount < job.MaxRetries {
		job.ErrorType = fmt.Sprintf("%T", err)
		job.ErrorMessage = err.Error()

		nextRetry := currentTimeFloat() + retryDelay(job.RetryCount)

		w.redisQuery("ZADD", w.nsKey("retry"), fmt.Sprintf("%f", nextRetry), job.JSON())
	}
}

type runningJob struct {
	Queue     string `json:"queue"`
	Job       *Job   `json:"payload"`
	Timestamp int64  `json:"run_at"`
}

// TODO: make a lua script for this
func (w *WorkerConfig) logJobStart(job *Job, workerID string) {
	conn := w.redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SADD", w.nsKey("workers"), workerID)
	conn.Send("SETEX", w.nsKey("worker:"+workerID+":started"), keyExpiry, time.Now().UTC().String())
	payload := &runningJob{job.Queue, job, time.Now().Unix()}
	json, _ := json.Marshal(payload)
	conn.Send("SETEX", w.nsKey("worker:"+workerID), keyExpiry, json)
	_, err := conn.Do("EXEC")
	if err != nil {
		w.handleError(err)
	}

	job.StartTime = time.Now()
	log.Printf("event=job_start job_id=%s job_type=%s queue=%s worker_id=%s pid=%d", job.ID, job.Type, job.Queue, workerID, pid)
}

// TODO: make a lua script for this
func (w *WorkerConfig) logJobFinish(job *Job, workerID string, success bool) {
	log.Printf("event=job_finish job_id=%s job_type=%s queue=%s duration=%v success=%t worker_id=%s pid=%d", job.ID, job.Type, job.Queue, time.Since(job.StartTime), success, workerID, pid)

	conn := w.redisPool.Get()
	defer conn.Close()

	conn.Send("MULTI")
	conn.Send("SREM", w.nsKey("workers"), workerID)
	conn.Send("DEL", w.nsKey("worker:"+workerID+":started"))
	conn.Send("DEL", w.nsKey("worker:"+workerID))
	conn.Send("INCR", w.nsKey("stat:processed"))
	if !success {
		conn.Send("INCR", w.nsKey("stat:failed"))
	}
	_, err := conn.Do("EXEC")
	if err != nil {
		w.handleError(err)
	}
}

func (w *WorkerConfig) nsKey(key string) string {
	if w.RedisNamespace != "" {
		return w.RedisNamespace + ":" + key
	}
	return key
}

// formula from Sidekiq (originally from delayed_job)
func retryDelay(count int) float64 {
	return math.Pow(float64(count), 4) + 15 + (float64(rand.Intn(30)) * (float64(count) + 1))
}

func currentTimeFloat() float64 {
	return float64(time.Now().UnixNano()) / float64(time.Second)
}

func panicToError(err interface{}) error {
	if str, ok := err.(string); ok {
		return fmt.Errorf(str)
	}
	return err.(error)
}

var (
	pid         = os.Getpid()
	hostname, _ = os.Hostname()
)

func workerID(i int) string {
	return fmt.Sprintf("%s:%d-%d", hostname, pid, i)
}

func workerType(worker Worker) reflect.Type {
	return reflect.Indirect(reflect.ValueOf(worker)).Type()
}

type UnknownWorkerError struct{ Type string }

func (e UnknownWorkerError) Error() string {
	return "gokiq: Unknown worker type: " + e.Type
}
<|endoftext|>


<commit_before><commit_msg>Add comments.<commit_after><|endoftext|>


<commit_before>package util

import (
	"io/ioutil"
	"path/filepath"

	"github.com/jdkato/prose/internal/model"
)

// ReadDataFile reads data from a file, panicking on any errors.
func ReadDataFile(path string) []byte {
	p, err := filepath.Abs(path)
	CheckError(err)

	data, ferr := ioutil.ReadFile(p)
	CheckError(ferr)

	return data
}

// CheckError panics if `err` is not `nil`.
func CheckError(err error) {
	if err != nil {
		panic(err)
	}
}

// Min returns the minimum of `a` and `b`.
func Min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// IsPunct determines if a character is a punctuation symbol.
func IsPunct(c byte) bool {
	for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
		if c == r {
			return true
		}
	}
	return false
}

// IsSpace determines if a character is a whitespace character.
func IsSpace(c byte) bool {
	for _, r := range []byte("\t\n\r\f\v") {
		if c == r {
			return true
		}
	}
	return false
}

// IsLetter determines if a character is letter.
func IsLetter(c byte) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

// IsAlnum determines if a character is a letter or a digit.
func IsAlnum(c byte) bool {
	return (c >= '0' && c <= '9') || IsLetter(c)
}

// GetAsset returns the named Asset.
func GetAsset(name string) []byte {
	b, err := model.Asset("model/" + name)
	CheckError(err)
	return b
}
<commit_msg>fix [internal]: update model path<commit_after>package util

import (
	"io/ioutil"
	"path/filepath"

	"github.com/jdkato/prose/internal/model"
)

// ReadDataFile reads data from a file, panicking on any errors.
func ReadDataFile(path string) []byte {
	p, err := filepath.Abs(path)
	CheckError(err)

	data, ferr := ioutil.ReadFile(p)
	CheckError(ferr)

	return data
}

// CheckError panics if `err` is not `nil`.
func CheckError(err error) {
	if err != nil {
		panic(err)
	}
}

// Min returns the minimum of `a` and `b`.
func Min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// IsPunct determines if a character is a punctuation symbol.
func IsPunct(c byte) bool {
	for _, r := range []byte("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") {
		if c == r {
			return true
		}
	}
	return false
}

// IsSpace determines if a character is a whitespace character.
func IsSpace(c byte) bool {
	for _, r := range []byte("\t\n\r\f\v") {
		if c == r {
			return true
		}
	}
	return false
}

// IsLetter determines if a character is letter.
func IsLetter(c byte) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

// IsAlnum determines if a character is a letter or a digit.
func IsAlnum(c byte) bool {
	return (c >= '0' && c <= '9') || IsLetter(c)
}

// GetAsset returns the named Asset.
func GetAsset(name string) []byte {
	b, err := model.Asset("internal/model/" + name)
	CheckError(err)
	return b
}
<|endoftext|>