<commit_before>package ipvs

import (
    "encoding/binary"
    "bytes"
    "fmt"
    "log"
    "net"
    "github.com/hkwi/nlgo"
    "syscall"
)

type Service struct {
    // id
    Af          uint16
    Protocol    uint16
    Addr        net.IP
    Port        uint16
    FwMark      uint32

    // params
    SchedName   string
    Flags       IPVSFlags
    Timeout     uint32
    Netmask     uint32
}

type Dest struct {
    // id
    // TODO: IPVS_DEST_ATTR_ADDR_FAMILY
    Addr        net.IP
    Port        uint16

    // params
    FwdMethod   uint32
    Weight      uint32
    UThresh     uint32
    LThresh     uint32

    // info
    ActiveConns     uint32
    InactConns      uint32
    PersistConns    uint32
}

func unpack(buf []byte, out interface{}) error {
    return binary.Read(bytes.NewReader(buf), binary.BigEndian, out)
}

func pack (in interface{}) []byte {
    var buf bytes.Buffer

    if err := binary.Write(&buf, binary.BigEndian, in); err != nil {
        panic(err)
    }

    return buf.Bytes()
}

func packAddr (af uint16, addr net.IP) []byte {
    switch af {
        case syscall.AF_INET:   return ([]byte)(addr.To4())
        case syscall.AF_INET6:  return ([]byte)(addr.To16())
        default:
            panic(fmt.Errorf("ipvs:packAddr: unknown af=%d addr=%v", af, addr))
    }
}

func htons (value uint16) uint16 {
    return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8)
}

func (self *Service) unpack(attrs nlgo.AttrList) error {
    var addr []byte
    var flags []byte

    for _, attr := range attrs {
        switch attr.Field() {
        case IPVS_SVC_ATTR_AF:          self.Af = attr.Value.(uint16)
        case IPVS_SVC_ATTR_PROTOCOL:    self.Protocol = attr.Value.(uint16)
        case IPVS_SVC_ATTR_ADDR:        addr = attr.Value.([]byte)
        case IPVS_SVC_ATTR_PORT:        self.Port = attr.Value.(uint16)
        case IPVS_SVC_ATTR_FWMARK:      self.FwMark = attr.Value.(uint32)
        case IPVS_SVC_ATTR_SCHED_NAME:  self.SchedName = attr.Value.(string)
        case IPVS_SVC_ATTR_FLAGS:       flags = attr.Value.([]byte)
        case IPVS_SVC_ATTR_TIMEOUT:     self.Timeout = attr.Value.(uint32)
        case IPVS_SVC_ATTR_NETMASK:     self.Netmask = attr.Value.(uint32)
        }
    }

    switch self.Af {
    case syscall.AF_INET:
        self.Addr = (net.IP)(addr[:4])

    case syscall.AF_INET6:
        self.Addr = (net.IP)(addr[:16])

    default:
        return fmt.Errorf("ipvs:Client.ListServices: unknown service AF=%d ADDR=%v", self.Af, addr)
    }

    if err := unpack(flags, &self.Flags); err != nil {
        return fmt.Errorf("ipvs:Service.unpack: flags: %s", err)
    }

    return nil
}

func (self *Service) attrs(full bool) nlgo.AttrList {
    var attrs nlgo.AttrList

    if self.FwMark != 0 {
        attrs = append(attrs,
            nlattr(IPVS_SVC_ATTR_AF, self.Af),
            nlattr(IPVS_SVC_ATTR_FWMARK, self.FwMark),
        )
    } else if self.Protocol != 0 && self.Addr != nil && self.Port != 0 {
        addr := packAddr(self.Af, self.Addr)

        attrs = append(attrs,
            nlattr(IPVS_SVC_ATTR_AF, self.Af),
            nlattr(IPVS_SVC_ATTR_PROTOCOL, self.Protocol),
            nlattr(IPVS_SVC_ATTR_ADDR, addr),
            nlattr(IPVS_SVC_ATTR_PORT, htons(self.Port)),       // network-order when sending
        )
    } else {
        panic("Incomplete service id fields")
    }

    if full {
        attrs = append(attrs,
            nlattr(IPVS_SVC_ATTR_SCHED_NAME,    self.SchedName),
            nlattr(IPVS_SVC_ATTR_FLAGS,         pack(&self.Flags)),
            nlattr(IPVS_SVC_ATTR_TIMEOUT,       self.Timeout),
            nlattr(IPVS_SVC_ATTR_NETMASK,       self.Netmask),
        )
    }

    return attrs
}

func (self *Dest) attrs(service *Service, full bool) nlgo.AttrList {
    var attrs nlgo.AttrList

    attrs = append(attrs,
        nlattr(IPVS_DEST_ATTR_ADDR, packAddr(service.Af, self.Addr)),
        nlattr(IPVS_DEST_ATTR_PORT, htons(self.Port)),
    )

    if full {
        attrs = append(attrs,
            nlattr(IPVS_DEST_ATTR_FWD_METHOD,   self.FwdMethod),
            nlattr(IPVS_DEST_ATTR_WEIGHT,       self.Weight),
            nlattr(IPVS_DEST_ATTR_U_THRESH,     self.UThresh),
            nlattr(IPVS_DEST_ATTR_L_THRESH,     self.LThresh),
        )
    }

    return attrs
}

type cmd struct {
    serviceId   *Service
    serviceFull *Service

    destId      *Dest
    destFull    *Dest
}

func (self cmd) attrs() nlgo.AttrList {
    attrs := nlgo.AttrList{}

    if self.serviceId != nil {
        attrs = append(attrs, nlattr(IPVS_CMD_ATTR_SERVICE, self.serviceId.attrs(false)))
    }
    if self.serviceFull != nil {
        attrs = append(attrs, nlattr(IPVS_CMD_ATTR_SERVICE, self.serviceFull.attrs(true)))
    }

    if self.destId != nil {
        attrs = append(attrs, nlattr(IPVS_CMD_ATTR_DEST, self.destId.attrs(self.serviceId, false)))
    }
    if self.destFull != nil {
        attrs = append(attrs, nlattr(IPVS_CMD_ATTR_DEST, self.destFull.attrs(self.serviceId, true)))
    }

    return attrs
}

func (client *Client) GetInfo() error {
    return client.request(Request{Cmd: IPVS_CMD_GET_INFO}, client.queryParser(IPVS_CMD_SET_INFO, ipvs_info_policy, func (attrs nlgo.AttrList) error {
        version := attrs.Get(IPVS_INFO_ATTR_VERSION).(uint32)
        size := attrs.Get(IPVS_INFO_ATTR_CONN_TAB_SIZE).(uint32)

        log.Printf("ipvs:Client.GetInfo: IPVS version=%d.%d.%d, size=%d\n",
            (version >> 16) & 0xFF,
            (version >> 8)  & 0xFF,
            (version >> 0)  & 0xFF,
            size,
        )

        return nil
    }))
}

func (client *Client) Flush() error {
    return client.exec(Request{Cmd: IPVS_CMD_FLUSH})
}

func (client *Client) NewService(service Service) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_NEW_SERVICE,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceFull: &service}.attrs(),
    })
}

func (client *Client) SetService(service Service) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_SET_SERVICE,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceFull: &service}.attrs(),
    })
}

func (client *Client) DelService(service Service) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_DEL_SERVICE,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceId: &service}.attrs(),
    })
}

func (client *Client) ListServices() ([]Service, error) {
    services := make([]Service, 0)
    request := Request{
        Cmd:    IPVS_CMD_GET_SERVICE,
        Flags:  syscall.NLM_F_DUMP,
    }

    if err := client.request(request, client.queryParser(IPVS_CMD_NEW_SERVICE, ipvs_cmd_policy, func (cmdAttrs nlgo.AttrList) error {
        svcAttrs := cmdAttrs.Get(IPVS_CMD_ATTR_SERVICE).(nlgo.AttrList)

        //log.Printf("ipvs:Client.ListServices: svc=%+v\n", ipvs_service_policy.Dump(svc_attrs))

        service := Service{}

        if err := service.unpack(svcAttrs); err != nil {
            return err
        } else {
            services = append(services, service)
        }

        return nil
    })); err != nil {
        return services, err
    } else {
        return services, nil
    }
}

func (client *Client) NewDest(service Service, dest Dest) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_NEW_DEST,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceId: &service, destFull: &dest}.attrs(),
    })
}

func (client *Client) SetDest(service Service, dest Dest) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_SET_DEST,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceId: &service, destFull: &dest}.attrs(),
    })
}

func (client *Client) DelDest(service Service, dest Dest) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_DEL_DEST,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceId: &service, destId: &dest}.attrs(),
    })
}

func (client *Client) ListDests(service Service) (error) {
    request := Request{
        Cmd:    IPVS_CMD_GET_DEST,
        Flags:  syscall.NLM_F_DUMP,
        Policy: ipvs_cmd_policy,
        Attrs:  cmd{serviceId: &service}.attrs(),
    }

    return client.request(request, client.queryParser(IPVS_CMD_NEW_DEST, ipvs_cmd_policy, func (cmd_attrs nlgo.AttrList) error {
        log.Printf("ipvs:Client.ListDests: cmd=%+v\n", ipvs_cmd_policy.Dump(cmd_attrs))

        return nil
    }))
}

<commit_msg>ipvs: serivice port htons/ntos; dest unpacking, ListDests<commit_after>package ipvs

import (
    "encoding/binary"
    "bytes"
    "fmt"
    "log"
    "net"
    "github.com/hkwi/nlgo"
    "syscall"
)

type Service struct {
    // id
    Af          uint16
    Protocol    uint16
    Addr        net.IP
    Port        uint16
    FwMark      uint32

    // params
    SchedName   string
    Flags       IPVSFlags
    Timeout     uint32
    Netmask     uint32
}

type Dest struct {
    // id
    // TODO: IPVS_DEST_ATTR_ADDR_FAMILY
    Addr        net.IP
    Port        uint16

    // params
    FwdMethod   uint32
    Weight      uint32
    UThresh     uint32
    LThresh     uint32

    // info
    ActiveConns     uint32
    InactConns      uint32
    PersistConns    uint32
}

func unpack(buf []byte, out interface{}) error {
    return binary.Read(bytes.NewReader(buf), binary.BigEndian, out)
}

func unpackAddr (af uint16, buf []byte) (net.IP, error) {
    // XXX: validate length?
    switch af {
    case syscall.AF_INET:
        return (net.IP)(buf[:4]), nil

    case syscall.AF_INET6:
        return (net.IP)(buf[:16]), nil

    default:
        return nil, fmt.Errorf("ipvs: unknown af=%d addr=%v", af, buf)
    }
}

func pack (in interface{}) []byte {
    var buf bytes.Buffer

    if err := binary.Write(&buf, binary.BigEndian, in); err != nil {
        panic(err)
    }

    return buf.Bytes()
}

func packAddr (af uint16, addr net.IP) []byte {
    switch af {
        case syscall.AF_INET:   return ([]byte)(addr.To4())
        case syscall.AF_INET6:  return ([]byte)(addr.To16())
        default:
            panic(fmt.Errorf("ipvs:packAddr: unknown af=%d addr=%v", af, addr))
    }
}

func htons (value uint16) uint16 {
    return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8)
}
func ntohs (value uint16) uint16 {
    return ((value & 0x00ff) << 8) | ((value & 0xff00) >> 8)
}

func (self *Service) unpack(attrs nlgo.AttrList) error {
    var addr []byte
    var flags []byte

    for _, attr := range attrs {
        switch attr.Field() {
        case IPVS_SVC_ATTR_AF:          self.Af = attr.Value.(uint16)
        case IPVS_SVC_ATTR_PROTOCOL:    self.Protocol = attr.Value.(uint16)
        case IPVS_SVC_ATTR_ADDR:        addr = attr.Value.([]byte)
        case IPVS_SVC_ATTR_PORT:        self.Port = ntohs(attr.Value.(uint16))
        case IPVS_SVC_ATTR_FWMARK:      self.FwMark = attr.Value.(uint32)
        case IPVS_SVC_ATTR_SCHED_NAME:  self.SchedName = attr.Value.(string)
        case IPVS_SVC_ATTR_FLAGS:       flags = attr.Value.([]byte)
        case IPVS_SVC_ATTR_TIMEOUT:     self.Timeout = attr.Value.(uint32)
        case IPVS_SVC_ATTR_NETMASK:     self.Netmask = attr.Value.(uint32)
        }
    }

    if addrIP, err := unpackAddr(self.Af, addr); err != nil {
        return fmt.Errorf("ipvs:Service.unpack: addr: %s", err)
    } else {
        self.Addr = addrIP
    }

    if err := unpack(flags, &self.Flags); err != nil {
        return fmt.Errorf("ipvs:Service.unpack: flags: %s", err)
    }

    return nil
}

func (self *Service) attrs(full bool) nlgo.AttrList {
    var attrs nlgo.AttrList

    if self.FwMark != 0 {
        attrs = append(attrs,
            nlattr(IPVS_SVC_ATTR_AF, self.Af),
            nlattr(IPVS_SVC_ATTR_FWMARK, self.FwMark),
        )
    } else if self.Protocol != 0 && self.Addr != nil && self.Port != 0 {
        addr := packAddr(self.Af, self.Addr)

        attrs = append(attrs,
            nlattr(IPVS_SVC_ATTR_AF, self.Af),
            nlattr(IPVS_SVC_ATTR_PROTOCOL, self.Protocol),
            nlattr(IPVS_SVC_ATTR_ADDR, addr),
            nlattr(IPVS_SVC_ATTR_PORT, htons(self.Port)),
        )
    } else {
        panic("Incomplete service id fields")
    }

    if full {
        attrs = append(attrs,
            nlattr(IPVS_SVC_ATTR_SCHED_NAME,    self.SchedName),
            nlattr(IPVS_SVC_ATTR_FLAGS,         pack(&self.Flags)),
            nlattr(IPVS_SVC_ATTR_TIMEOUT,       self.Timeout),
            nlattr(IPVS_SVC_ATTR_NETMASK,       self.Netmask),
        )
    }

    return attrs
}

func (self *Dest) unpack(service Service, attrs nlgo.AttrList) error {
    var addr []byte

    for _, attr := range attrs {
        switch attr.Field() {
        case IPVS_DEST_ATTR_ADDR:       addr = attr.Value.([]byte)
        case IPVS_DEST_ATTR_PORT:       self.Port = ntohs(attr.Value.(uint16))
        case IPVS_DEST_ATTR_FWD_METHOD: self.FwdMethod = attr.Value.(uint32)
        case IPVS_DEST_ATTR_WEIGHT:     self.Weight = attr.Value.(uint32)
        case IPVS_DEST_ATTR_U_THRESH:   self.UThresh = attr.Value.(uint32)
        case IPVS_DEST_ATTR_L_THRESH:   self.LThresh = attr.Value.(uint32)
        case IPVS_DEST_ATTR_ACTIVE_CONNS:   self.ActiveConns = attr.Value.(uint32)
        case IPVS_DEST_ATTR_INACT_CONNS:    self.InactConns = attr.Value.(uint32)
        case IPVS_DEST_ATTR_PERSIST_CONNS:  self.PersistConns = attr.Value.(uint32)
        }
    }

    if addrIP, err := unpackAddr(service.Af, addr); err != nil {
        return fmt.Errorf("ipvs:Dest.unpack: addr: %s", err)
    } else {
        self.Addr = addrIP
    }

    return nil
}

func (self *Dest) attrs(service *Service, full bool) nlgo.AttrList {
    var attrs nlgo.AttrList

    attrs = append(attrs,
        nlattr(IPVS_DEST_ATTR_ADDR, packAddr(service.Af, self.Addr)),
        nlattr(IPVS_DEST_ATTR_PORT, htons(self.Port)),
    )

    if full {
        attrs = append(attrs,
            nlattr(IPVS_DEST_ATTR_FWD_METHOD,   self.FwdMethod),
            nlattr(IPVS_DEST_ATTR_WEIGHT,       self.Weight),
            nlattr(IPVS_DEST_ATTR_U_THRESH,     self.UThresh),
            nlattr(IPVS_DEST_ATTR_L_THRESH,     self.LThresh),
        )
    }

    return attrs
}

type cmd struct {
    serviceId   *Service
    serviceFull *Service

    destId      *Dest
    destFull    *Dest
}

func (self cmd) attrs() nlgo.AttrList {
    attrs := nlgo.AttrList{}

    if self.serviceId != nil {
        attrs = append(attrs, nlattr(IPVS_CMD_ATTR_SERVICE, self.serviceId.attrs(false)))
    }
    if self.serviceFull != nil {
        attrs = append(attrs, nlattr(IPVS_CMD_ATTR_SERVICE, self.serviceFull.attrs(true)))
    }

    if self.destId != nil {
        attrs = append(attrs, nlattr(IPVS_CMD_ATTR_DEST, self.destId.attrs(self.serviceId, false)))
    }
    if self.destFull != nil {
        attrs = append(attrs, nlattr(IPVS_CMD_ATTR_DEST, self.destFull.attrs(self.serviceId, true)))
    }

    return attrs
}

func (client *Client) GetInfo() error {
    return client.request(Request{Cmd: IPVS_CMD_GET_INFO}, client.queryParser(IPVS_CMD_SET_INFO, ipvs_info_policy, func (attrs nlgo.AttrList) error {
        version := attrs.Get(IPVS_INFO_ATTR_VERSION).(uint32)
        size := attrs.Get(IPVS_INFO_ATTR_CONN_TAB_SIZE).(uint32)

        log.Printf("ipvs:Client.GetInfo: IPVS version=%d.%d.%d, size=%d\n",
            (version >> 16) & 0xFF,
            (version >> 8)  & 0xFF,
            (version >> 0)  & 0xFF,
            size,
        )

        return nil
    }))
}

func (client *Client) Flush() error {
    return client.exec(Request{Cmd: IPVS_CMD_FLUSH})
}

func (client *Client) NewService(service Service) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_NEW_SERVICE,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceFull: &service}.attrs(),
    })
}

func (client *Client) SetService(service Service) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_SET_SERVICE,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceFull: &service}.attrs(),
    })
}

func (client *Client) DelService(service Service) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_DEL_SERVICE,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceId: &service}.attrs(),
    })
}

func (client *Client) ListServices() ([]Service, error) {
    services := make([]Service, 0)
    request := Request{
        Cmd:    IPVS_CMD_GET_SERVICE,
        Flags:  syscall.NLM_F_DUMP,
    }

    if err := client.request(request, client.queryParser(IPVS_CMD_NEW_SERVICE, ipvs_cmd_policy, func (cmdAttrs nlgo.AttrList) error {
        svcAttrs := cmdAttrs.Get(IPVS_CMD_ATTR_SERVICE).(nlgo.AttrList)

        //log.Printf("ipvs:Client.ListServices: svc=%+v\n", ipvs_service_policy.Dump(svc_attrs))

        service := Service{}

        if err := service.unpack(svcAttrs); err != nil {
            return err
        } else {
            services = append(services, service)
        }

        return nil
    })); err != nil {
        return services, err
    } else {
        return services, nil
    }
}

func (client *Client) NewDest(service Service, dest Dest) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_NEW_DEST,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceId: &service, destFull: &dest}.attrs(),
    })
}

func (client *Client) SetDest(service Service, dest Dest) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_SET_DEST,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceId: &service, destFull: &dest}.attrs(),
    })
}

func (client *Client) DelDest(service Service, dest Dest) error {
    return client.exec(Request{
        Cmd:        IPVS_CMD_DEL_DEST,
        Policy:     ipvs_cmd_policy,
        Attrs:      cmd{serviceId: &service, destId: &dest}.attrs(),
    })
}

func (client *Client) ListDests(service Service) ([]Dest, error) {
    dests := make([]Dest, 0)
    request := Request{
        Cmd:    IPVS_CMD_GET_DEST,
        Flags:  syscall.NLM_F_DUMP,
        Policy: ipvs_cmd_policy,
        Attrs:  cmd{serviceId: &service}.attrs(),
    }

    if err := client.request(request, client.queryParser(IPVS_CMD_NEW_DEST, ipvs_cmd_policy, func (cmdAttrs nlgo.AttrList) error {
        destAttrs := cmdAttrs.Get(IPVS_CMD_ATTR_DEST).(nlgo.AttrList)

        log.Printf("ipvs:Client.ListDests: dest=%+v\n", ipvs_dest_policy.Dump(destAttrs))

        dest := Dest{}

        if err := dest.unpack(service, destAttrs); err != nil {
            return err
        } else {
            dests = append(dests, dest)
        }

        return nil
    })); err != nil {
        return nil, err
    } else {
        return dests, nil
    }
}

<|endoftext|>


<commit_before>package kubernetes

import (
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strings"

	"github.com/weaveworks/fluxy/registry"
)

// UpdatePodController takes the body of a ReplicationController or Deployment
// resource definition (specified in YAML) and the name of the new image that
// should be put in the definition (in the format "repo.org/group/name:tag"). It
// returns a new resource definition body where all references to the old image
// have been replaced with the new one.
//
// This function has many additional requirements that are likely in flux. Read
// the source to learn about them.
func UpdatePodController(def []byte, newImageName string, trace io.Writer) ([]byte, error) {
	var buf bytes.Buffer
	err := tryUpdate(string(def), newImageName, trace, &buf)
	return buf.Bytes(), err
}

// Attempt to update an RC or Deployment config. This makes several assumptions
// that are justified only with the phrase "because that's how we do it",
// including:
//
//  * the file is a replication controller or deployment
//  * the update is from one tag of an image to another tag of the
//    same image; e.g., "weaveworks/helloworld:a00001" to
//    "weaveworks/helloworld:a00002"
//  * the container spec to update is the (first) one that uses the
//    same image name (e.g., weaveworks/helloworld)
//  * the name of the controller is updated to reflect the new tag
//  * there's a label which must be updated in both the pod spec and the selector
//  * the file uses canonical YAML syntax, that is, one line per item
//  * ... other assumptions as encoded in the regular expressions used
//
// Here's an example of the assumed structure:
//
// ```
// apiVersion: v1
// kind: ReplicationController # not presently checked
// metadata:                         # )
//   name: helloworld-master-a000001 # ) this structure, and naming scheme, are assumed
// spec:
//   replicas: 2
//   selector:                 # )
//     name: helloworld        # ) this use of labels is assumed
//     version: master-a000001 # )
//   template:
//     metadata:
//       labels:                   # )
//         name: helloworld        # ) this structure is assumed, as for the selector
//         version: master-a000001 # )
//     spec:
//       containers:
//       # extra container specs are allowed here ...
//       - name: helloworld                                    # )
//         image: quay.io/weaveworks/helloworld:master-a000001 # ) these must be together
//         args:
//         - -msg=Ahoy
//         ports:
//         - containerPort: 80
// ```
func tryUpdate(def, newImageStr string, trace io.Writer, out io.Writer) error {
	newImage := registry.ParseImage(newImageStr)

	nameRE := multilineRE(
		`metadata:\s*`,
		`  name:\s*"?([\w-]+)"?\s*`,
	)
	matches := nameRE.FindStringSubmatch(def)
	if matches == nil || len(matches) < 2 {
		return fmt.Errorf("Could not find resource name")
	}
	oldDefName := matches[1]
	fmt.Fprintf(trace, "Found resource name %q in fragment:\n\n%s\n\n", oldDefName, matches[0])

	imageRE := multilineRE(
		`      containers:.*`,
		`(?:      .*\n)*(?:  ){3,4}- name:\s*"?([\w-]+)"?(?:\s.*)?`,
		`(?:  ){4,5}image:\s*"?(`+newImage.Repository()+`:[\w-]+)"?(\s.*)?`,
	)

	matches = imageRE.FindStringSubmatch(def)
	if matches == nil || len(matches) < 3 {
		return fmt.Errorf("Could not find image name")
	}
	containerName := matches[1]
	oldImage := registry.ParseImage(matches[2])
	fmt.Fprintf(trace, "Found container %q using image %v in fragment:\n\n%s\n\n", containerName, oldImage, matches[0])

	if oldImage.Repository() != newImage.Repository() {
		return fmt.Errorf(`expected existing image name and new image name to match, but %q != %q`, oldImage.Repository(), newImage.Repository())
	}

	// Now to replace bits. Specifically,
	// * the name, with a re-tagged name
	// * the image for the container
	// * the version label (in two places)
	//
	// Some values (most likely the version) will be interpreted as a
	// number if unquoted; while, on the other hand, it is apparently
	// not OK to quote things that don't look like numbers. So: we
	// extract values *without* quotes, and add them if necessary.

	newDefName := oldDefName
	if strings.HasSuffix(oldDefName, oldImage.Tag) {
		newDefName = oldDefName[:len(oldDefName)-len(oldImage.Tag)] + newImage.Tag
	}

	newDefName = maybeQuote(newDefName)
	newTag := maybeQuote(newImage.Tag)

	fmt.Fprintln(trace, "")
	fmt.Fprintln(trace, "Replacing ...")
	fmt.Fprintf(trace, "Resource name: %s -> %s\n", oldDefName, newDefName)
	fmt.Fprintf(trace, "Version in templates (and selector if present): %s -> %s\n", oldImage.Tag, newTag)
	fmt.Fprintf(trace, "Image in templates: %s -> %s\n", oldImage, newImage)
	fmt.Fprintln(trace, "")

	// The name we want is that under `metadata:`, which will be indented once
	replaceRCNameRE := regexp.MustCompile(`(?m:^(  name:\s*) (?:"?[\w-]+"?)(\s.*)$)`)
	withNewDefName := replaceRCNameRE.ReplaceAllString(def, fmt.Sprintf(`$1 %s$2`, newDefName))

	// Replacing labels: these are in two places, the container template and the selector
	replaceLabelsRE := multilineRE(
		`((?:  selector|      labels):.*)`,
		`((?:  ){2,4}name:.*)`,
		`((?:  ){2,4}version:\s*) (?:"?[-\w]+"?)(\s.*)`,
	)
	replaceLabels := fmt.Sprintf("$1\n$2\n$3 %s$4", newTag)
	withNewLabels := replaceLabelsRE.ReplaceAllString(withNewDefName, replaceLabels)

	replaceImageRE := multilineRE(
		`((?:  ){3,4}- name:\s*`+containerName+`)`,
		`((?:  ){4,5}image:\s*) .*`,
	)
	replaceImage := fmt.Sprintf("$1\n$2 %s$3", newImage.String())
	withNewImage := replaceImageRE.ReplaceAllString(withNewLabels, replaceImage)

	fmt.Fprint(out, withNewImage)
	return nil
}

func multilineRE(lines ...string) *regexp.Regexp {
	return regexp.MustCompile(`(?m:^` + strings.Join(lines, "\n") + `$)`)
}

var looksLikeNumber *regexp.Regexp = regexp.MustCompile("^(" + strings.Join([]string{
	`(-?[1-9](\.[0-9]*[1-9])?(e[-+][1-9][0-9]*)?)`,
	`(-?(0|[1-9][0-9]*))`,
	`(0|(\.inf)|(-\.inf)|(\.nan))`},
	"|") + ")$")

func maybeQuote(scalar string) string {
	fmt.Printf("Looks like number: %s = %v\n", scalar, looksLikeNumber.MatchString(scalar))
	if looksLikeNumber.MatchString(scalar) {
		return `"` + scalar + `"`
	}
	return scalar
}
<commit_msg>Remove a no-longer neeeded debug aid<commit_after>package kubernetes

import (
	"bytes"
	"fmt"
	"io"
	"regexp"
	"strings"

	"github.com/weaveworks/fluxy/registry"
)

// UpdatePodController takes the body of a ReplicationController or Deployment
// resource definition (specified in YAML) and the name of the new image that
// should be put in the definition (in the format "repo.org/group/name:tag"). It
// returns a new resource definition body where all references to the old image
// have been replaced with the new one.
//
// This function has many additional requirements that are likely in flux. Read
// the source to learn about them.
func UpdatePodController(def []byte, newImageName string, trace io.Writer) ([]byte, error) {
	var buf bytes.Buffer
	err := tryUpdate(string(def), newImageName, trace, &buf)
	return buf.Bytes(), err
}

// Attempt to update an RC or Deployment config. This makes several assumptions
// that are justified only with the phrase "because that's how we do it",
// including:
//
//  * the file is a replication controller or deployment
//  * the update is from one tag of an image to another tag of the
//    same image; e.g., "weaveworks/helloworld:a00001" to
//    "weaveworks/helloworld:a00002"
//  * the container spec to update is the (first) one that uses the
//    same image name (e.g., weaveworks/helloworld)
//  * the name of the controller is updated to reflect the new tag
//  * there's a label which must be updated in both the pod spec and the selector
//  * the file uses canonical YAML syntax, that is, one line per item
//  * ... other assumptions as encoded in the regular expressions used
//
// Here's an example of the assumed structure:
//
// ```
// apiVersion: v1
// kind: ReplicationController # not presently checked
// metadata:                         # )
//   name: helloworld-master-a000001 # ) this structure, and naming scheme, are assumed
// spec:
//   replicas: 2
//   selector:                 # )
//     name: helloworld        # ) this use of labels is assumed
//     version: master-a000001 # )
//   template:
//     metadata:
//       labels:                   # )
//         name: helloworld        # ) this structure is assumed, as for the selector
//         version: master-a000001 # )
//     spec:
//       containers:
//       # extra container specs are allowed here ...
//       - name: helloworld                                    # )
//         image: quay.io/weaveworks/helloworld:master-a000001 # ) these must be together
//         args:
//         - -msg=Ahoy
//         ports:
//         - containerPort: 80
// ```
func tryUpdate(def, newImageStr string, trace io.Writer, out io.Writer) error {
	newImage := registry.ParseImage(newImageStr)

	nameRE := multilineRE(
		`metadata:\s*`,
		`  name:\s*"?([\w-]+)"?\s*`,
	)
	matches := nameRE.FindStringSubmatch(def)
	if matches == nil || len(matches) < 2 {
		return fmt.Errorf("Could not find resource name")
	}
	oldDefName := matches[1]
	fmt.Fprintf(trace, "Found resource name %q in fragment:\n\n%s\n\n", oldDefName, matches[0])

	imageRE := multilineRE(
		`      containers:.*`,
		`(?:      .*\n)*(?:  ){3,4}- name:\s*"?([\w-]+)"?(?:\s.*)?`,
		`(?:  ){4,5}image:\s*"?(`+newImage.Repository()+`:[\w-]+)"?(\s.*)?`,
	)

	matches = imageRE.FindStringSubmatch(def)
	if matches == nil || len(matches) < 3 {
		return fmt.Errorf("Could not find image name")
	}
	containerName := matches[1]
	oldImage := registry.ParseImage(matches[2])
	fmt.Fprintf(trace, "Found container %q using image %v in fragment:\n\n%s\n\n", containerName, oldImage, matches[0])

	if oldImage.Repository() != newImage.Repository() {
		return fmt.Errorf(`expected existing image name and new image name to match, but %q != %q`, oldImage.Repository(), newImage.Repository())
	}

	// Now to replace bits. Specifically,
	// * the name, with a re-tagged name
	// * the image for the container
	// * the version label (in two places)
	//
	// Some values (most likely the version) will be interpreted as a
	// number if unquoted; while, on the other hand, it is apparently
	// not OK to quote things that don't look like numbers. So: we
	// extract values *without* quotes, and add them if necessary.

	newDefName := oldDefName
	if strings.HasSuffix(oldDefName, oldImage.Tag) {
		newDefName = oldDefName[:len(oldDefName)-len(oldImage.Tag)] + newImage.Tag
	}

	newDefName = maybeQuote(newDefName)
	newTag := maybeQuote(newImage.Tag)

	fmt.Fprintln(trace, "")
	fmt.Fprintln(trace, "Replacing ...")
	fmt.Fprintf(trace, "Resource name: %s -> %s\n", oldDefName, newDefName)
	fmt.Fprintf(trace, "Version in templates (and selector if present): %s -> %s\n", oldImage.Tag, newTag)
	fmt.Fprintf(trace, "Image in templates: %s -> %s\n", oldImage, newImage)
	fmt.Fprintln(trace, "")

	// The name we want is that under `metadata:`, which will be indented once
	replaceRCNameRE := regexp.MustCompile(`(?m:^(  name:\s*) (?:"?[\w-]+"?)(\s.*)$)`)
	withNewDefName := replaceRCNameRE.ReplaceAllString(def, fmt.Sprintf(`$1 %s$2`, newDefName))

	// Replacing labels: these are in two places, the container template and the selector
	replaceLabelsRE := multilineRE(
		`((?:  selector|      labels):.*)`,
		`((?:  ){2,4}name:.*)`,
		`((?:  ){2,4}version:\s*) (?:"?[-\w]+"?)(\s.*)`,
	)
	replaceLabels := fmt.Sprintf("$1\n$2\n$3 %s$4", newTag)
	withNewLabels := replaceLabelsRE.ReplaceAllString(withNewDefName, replaceLabels)

	replaceImageRE := multilineRE(
		`((?:  ){3,4}- name:\s*`+containerName+`)`,
		`((?:  ){4,5}image:\s*) .*`,
	)
	replaceImage := fmt.Sprintf("$1\n$2 %s$3", newImage.String())
	withNewImage := replaceImageRE.ReplaceAllString(withNewLabels, replaceImage)

	fmt.Fprint(out, withNewImage)
	return nil
}

func multilineRE(lines ...string) *regexp.Regexp {
	return regexp.MustCompile(`(?m:^` + strings.Join(lines, "\n") + `$)`)
}

var looksLikeNumber *regexp.Regexp = regexp.MustCompile("^(" + strings.Join([]string{
	`(-?[1-9](\.[0-9]*[1-9])?(e[-+][1-9][0-9]*)?)`,
	`(-?(0|[1-9][0-9]*))`,
	`(0|(\.inf)|(-\.inf)|(\.nan))`},
	"|") + ")$")

func maybeQuote(scalar string) string {
	if looksLikeNumber.MatchString(scalar) {
		return `"` + scalar + `"`
	}
	return scalar
}
<|endoftext|>


<commit_before>package archiver_test

import (
	"io/ioutil"
	"os"
	"path"
	"testing"

	"github.com/mholt/archiver"
)

func requireRegularFile(t *testing.T, path string) os.FileInfo {
	fileInfo, err := os.Stat(path)
	if err != nil {
		t.Fatalf("fileInfo on '%s': %v", path, err)
	}

	if !fileInfo.Mode().IsRegular() {
		t.Fatalf("'%s' expected to be a regular file", path)
	}

	return fileInfo
}

func assertSameFile(t *testing.T, f1, f2 os.FileInfo) {
	if !os.SameFile(f1, f2) {
		t.Errorf("expected '%s' and '%s' to be the same file", f1.Name(), f2.Name())
	}
}

func TestDefaultTar_Unarchive_HardlinkSuccess(t *testing.T) {
	source := "testdata/gnu-hardlinks.tar"

	destination, err := ioutil.TempDir("", "archiver_tar_test")
	if err != nil {
		t.Fatalf("creating temp dir: %v", err)
	}
	defer os.RemoveAll(destination)

	err = archiver.DefaultTar.Unarchive(source, destination)
	if err != nil {
		t.Fatalf("unarchiving '%s' to '%s': %v", source, destination, err)
	}

	fileaInfo := requireRegularFile(t, path.Join(destination, "dir-1", "dir-2", "file-a"))
	filebInfo := requireRegularFile(t, path.Join(destination, "dir-1", "dir-2", "file-b"))
	assertSameFile(t, fileaInfo, filebInfo)
}

func TestDefaultTar_Extract_HardlinkSuccess(t *testing.T) {
	source := "testdata/gnu-hardlinks.tar"

	destination, err := ioutil.TempDir("", "archiver_tar_test")
	if err != nil {
		t.Fatalf("creating temp dir: %v", err)
	}
	defer os.RemoveAll(destination)

	err = archiver.DefaultTar.Extract(source, path.Join("dir-1", "dir-2"), destination)
	if err != nil {
		t.Fatalf("unarchiving '%s' to '%s': %v", source, destination, err)
	}

	fileaInfo := requireRegularFile(t, path.Join(destination, "dir-2", "file-a"))
	filebInfo := requireRegularFile(t, path.Join(destination, "dir-2", "file-b"))
	assertSameFile(t, fileaInfo, filebInfo)
}
<commit_msg>Fix import path in test (#197)<commit_after>package archiver_test

import (
	"io/ioutil"
	"os"
	"path"
	"testing"

	"github.com/mholt/archiver/v3"
)

func requireRegularFile(t *testing.T, path string) os.FileInfo {
	fileInfo, err := os.Stat(path)
	if err != nil {
		t.Fatalf("fileInfo on '%s': %v", path, err)
	}

	if !fileInfo.Mode().IsRegular() {
		t.Fatalf("'%s' expected to be a regular file", path)
	}

	return fileInfo
}

func assertSameFile(t *testing.T, f1, f2 os.FileInfo) {
	if !os.SameFile(f1, f2) {
		t.Errorf("expected '%s' and '%s' to be the same file", f1.Name(), f2.Name())
	}
}

func TestDefaultTar_Unarchive_HardlinkSuccess(t *testing.T) {
	source := "testdata/gnu-hardlinks.tar"

	destination, err := ioutil.TempDir("", "archiver_tar_test")
	if err != nil {
		t.Fatalf("creating temp dir: %v", err)
	}
	defer os.RemoveAll(destination)

	err = archiver.DefaultTar.Unarchive(source, destination)
	if err != nil {
		t.Fatalf("unarchiving '%s' to '%s': %v", source, destination, err)
	}

	fileaInfo := requireRegularFile(t, path.Join(destination, "dir-1", "dir-2", "file-a"))
	filebInfo := requireRegularFile(t, path.Join(destination, "dir-1", "dir-2", "file-b"))
	assertSameFile(t, fileaInfo, filebInfo)
}

func TestDefaultTar_Extract_HardlinkSuccess(t *testing.T) {
	source := "testdata/gnu-hardlinks.tar"

	destination, err := ioutil.TempDir("", "archiver_tar_test")
	if err != nil {
		t.Fatalf("creating temp dir: %v", err)
	}
	defer os.RemoveAll(destination)

	err = archiver.DefaultTar.Extract(source, path.Join("dir-1", "dir-2"), destination)
	if err != nil {
		t.Fatalf("unarchiving '%s' to '%s': %v", source, destination, err)
	}

	fileaInfo := requireRegularFile(t, path.Join(destination, "dir-2", "file-a"))
	filebInfo := requireRegularFile(t, path.Join(destination, "dir-2", "file-b"))
	assertSameFile(t, fileaInfo, filebInfo)
}
<|endoftext|>


<commit_before>// machine description of iris1
package iris1

import (
	"fmt"
)

const (
	RegisterCount            = 256
	MemorySize               = 65536
	MajorOperationGroupCount = 8
	SystemCallCount          = 256
)
const (
	// reserved registers
	FalseRegister = iota
	TrueRegister
	InstructionPointer
	StackPointer
	PredicateRegister
	CountRegister
	CallPointer
	UserRegisterBegin
)

const (
	// Error codes
	ErrorNone = iota
	ErrorPanic
	ErrorGetRegisterOutOfRange
	ErrorPutRegisterOutOfRange
	ErrorInvalidInstructionGroupProvided
	ErrorInvalidArithmeticOperation
	ErrorInvalidMoveOperation
	ErrorInvalidJumpOperation
	ErrorInvalidCompareOperation
	ErrorInvalidMiscOperation
	ErrorInvalidSystemCommand
	ErrorWriteToFalseRegister
	ErrorWriteToTrueRegister
	ErrorEncodeByteOutOfRange
	ErrorGroupValueOutOfRange
	ErrorOpValueOutOfRange
)
const (
	// Instruction groups
	InstructionGroupArithmetic = iota
	InstructionGroupMove
	InstructionGroupJump
	InstructionGroupCompare
	InstructionGroupMisc
)

var errorLookup = []string{
	"None",
	"The core was sent a panic signal with argument %d!",
	"Attempted to get the value of invalid register r%d",
	"Attempted to set the value of invalid register r%d",
	"Instruction group %d is not a valid instruction group!",
	"Illegal arithmetic operation %d",
	"Illegal move operation %d",
	"Illegal jump operation %d",
	"Illegal compare operation %d",
	"Illegal misc operation %d",
	"Invalid system command %d",
	"Attempted to write %d to false register",
	"Attempted to write %d to true register!",
	"Specified illegal byte offset %d to encode data into",
	"Provided group id %d is larger than the space allotted to specifying the group",
	"Provided op id %d is larger than the space allotted to specifying the op",
}

type Word uint16
type Dword uint32
type Instruction Dword

func (this Instruction) group() byte {
	return byte(((this & 0x000000FF) & 0x7))
}
func (this Instruction) op() byte {
	return byte(((this & 0x000000FF) & 0xF8) >> 3)
}
func (this Instruction) register(index int) (byte, error) {
	switch index {
	case 0:
		return byte(this), nil
	case 1:
		return byte((this & 0x0000FF00) >> 8), nil
	case 2:
		return byte((this & 0x00FF0000) >> 16), nil
	case 3:
		return byte((this & 0xFF000000) >> 24), nil
	default:
		return 0, fmt.Errorf("Register index: %d is out of range!", index)
	}
}

func (this Instruction) immediate() Word {
	return Word((this & 0xFFFF0000) >> 16)
}

func (this *Instruction) setGroup(group byte) {
	*this = ((*this &^ 0x7) | Instruction(group))
}
func (this *Instruction) setOp(op byte) {
	*this = ((*this &^ 0xF8) | (Instruction(op) << 3))
}
func (this *Instruction) setByte(index int, value byte) error {
	switch index {
	case 1:
		*this = ((*this &^ 0x0000FF00) | (Instruction(value) << 8))
	case 2:
		*this = ((*this &^ 0x00FF0000) | (Instruction(value) << 16))
	case 3:
		*this = ((*this &^ 0xFF000000) | (Instruction(value) << 24))
	default:
		return NewError(ErrorEncodeByteOutOfRange, uint(index))
	}
	return nil
}
func (this *Instruction) setImmediate(value Word) {
	*this = ((*this &^ 0xFFFF0000) | (Instruction(value) << 16))
}

type DecodedInstruction struct {
	Group byte
	Op    byte
	Data  [3]byte
}

func (this Instruction) Decode() (*DecodedInstruction, error) {
	var di DecodedInstruction
	di.Group = this.group()
	di.Op = this.op()
	if value, err := this.register(1); err != nil {
		return nil, err
	} else {
		di.Data[0] = value
	}
	if value, err := this.register(2); err != nil {
		return nil, err
	} else {
		di.Data[1] = value
	}
	if value, err := this.register(3); err != nil {
		return nil, err
	} else {
		di.Data[2] = value
	}
	return &di, nil
}

func (this *DecodedInstruction) SetImmediate(value Word) {
	this.Data[1] = byte(value)
	this.Data[2] = byte(value >> 8)
}
func (this *DecodedInstruction) Immediate() Word {
	return Word((Word(this.Data[2]) << 8) | Word(this.Data[1]))
}

func (this *DecodedInstruction) Encode() *Instruction {
	i := new(Instruction)
	// encode group
	i.setGroup(this.Group)
	i.setOp(this.Op)
	i.setByte(1, this.Data[0])
	i.setByte(2, this.Data[1])
	i.setByte(3, this.Data[2])
	return i
}

type IrisError struct {
	value, code uint
}

func NewError(code, value uint) error {
	return &IrisError{code: code, value: value}
}

func (this IrisError) Error() string {
	if this.code == 0 {
		return fmt.Sprintf("No Error with value %d!!! This should never ever showup!", this.value)
	} else if this.code >= uint(len(errorLookup)) {
		return fmt.Sprintf("Unknown error %d with value %d! Something really bad happened!", this.code, this.value)
	} else {
		return fmt.Sprintf(errorLookup[this.code], this.value)
	}
}

type ExecutionUnit func(*Core, *DecodedInstruction) error
type SystemCall ExecutionUnit

type Core struct {
	gpr   [RegisterCount - UserRegisterBegin]Word
	code  [MemorySize]Instruction
	data  [MemorySize]Word
	stack [MemorySize]Word
	call  [MemorySize]Word
	// internal registers that should be easy to find
	instructionPointer Word
	stackPointer       Word
	callPointer        Word
	count              Word
	predicate          Word
	advancePc          bool
	terminateExecution bool
	groups             [MajorOperationGroupCount]ExecutionUnit
	systemCalls        [SystemCallCount]SystemCall
}

func (this *Core) SetRegister(index byte, value Word) error {
	switch index {
	case FalseRegister:
		return NewError(ErrorWriteToFalseRegister, uint(value))
	case TrueRegister:
		return NewError(ErrorWriteToTrueRegister, uint(value))
	case InstructionPointer:
		this.instructionPointer = value
	case StackPointer:
		this.stackPointer = value
	case PredicateRegister:
		this.predicate = value
	case CountRegister:
		this.count = value
	case CallPointer:
		this.callPointer = value
	default:
		this.gpr[index-UserRegisterBegin] = value
	}
	return nil
}
func (this *Core) Register(index byte) Word {
	switch index {
	case FalseRegister:
		return 0
	case TrueRegister:
		return 1
	case InstructionPointer:
		return this.instructionPointer
	case StackPointer:
		return this.stackPointer
	case PredicateRegister:
		return this.predicate
	case CountRegister:
		return this.count
	case CallPointer:
		return this.callPointer
	default:
		// do the offset calculation
		return this.gpr[index-UserRegisterBegin]
	}
}

func (this *Core) CodeMemory(address Word) Instruction {
	return this.code[address]
}
func (this *Core) SetCodeMemory(address Word, value Instruction) error {
	this.code[address] = value
	return nil
}
func (this *Core) Call(addr Word) error {
	this.callPointer++
	this.call[this.callPointer] = this.NextInstructionAddress()
	return this.SetRegister(InstructionPointer, addr)
}
func (this *Core) Return() Word {
	value := this.call[this.callPointer]
	this.callPointer--
	return value
}
func (this *Core) Push(value Word) {
	this.stackPointer++
	this.stack[this.stackPointer] = value
}
func (this *Core) Peek() Word {
	return this.stack[this.stackPointer]
}
func (this *Core) Pop() Word {
	value := this.stack[this.stackPointer]
	this.stackPointer--
	return value
}
func (this *Core) DataMemory(address Word) Word {
	return this.data[address]
}
func (this *Core) SetDataMemory(address, value Word) error {
	this.data[address] = value
	return nil
}

var defaultExecutionUnits = []struct {
	Group byte
	Unit  ExecutionUnit
}{
	{Group: InstructionGroupArithmetic, Unit: arithmetic},
	{Group: InstructionGroupMove, Unit: move},
	{Group: InstructionGroupJump, Unit: jump},
	{Group: InstructionGroupCompare, Unit: compare},
	{Group: InstructionGroupMisc, Unit: misc},
}

func New() (*Core, error) {
	var c Core
	c.advancePc = true
	c.terminateExecution = false
	if err := c.SetRegister(InstructionPointer, 0); err != nil {
		return nil, err
	}
	if err := c.SetRegister(PredicateRegister, 0); err != nil {
		return nil, err
	}
	if err := c.SetRegister(StackPointer, 0xFFFF); err != nil {
		return nil, err
	}
	if err := c.SetRegister(CallPointer, 0xFFFF); err != nil {
		return nil, err
	}
	if err := c.SetRegister(CountRegister, 0); err != nil {
		return nil, err
	}
	for i := 0; i < MajorOperationGroupCount; i++ {
		if err := c.InstallExecutionUnit(byte(i), defaultExtendedUnit); err != nil {
			return nil, err
		}
	}
	for _, unit := range defaultExecutionUnits {
		if err := c.InstallExecutionUnit(unit.Group, unit.Unit); err != nil {
			return nil, err
		}
	}
	for i := 0; i < SystemCallCount; i++ {
		if err := c.InstallSystemCall(byte(i), defaultSystemCall); err != nil {
			return nil, err
		}
	}
	c.InstallSystemCall(SystemCallTerminate, terminateSystemCall)
	c.InstallSystemCall(SystemCallPanic, panicSystemCall)
	return &c, nil
}

func (this *Core) InstallExecutionUnit(group byte, fn ExecutionUnit) error {
	if group >= MajorOperationGroupCount {
		return NewError(ErrorGroupValueOutOfRange, uint(group))
	} else {
		this.groups[group] = fn
		return nil
	}
}
func (this *Core) Invoke(inst *DecodedInstruction) error {
	return this.groups[inst.Group](this, inst)
}
func (this *Core) InstallSystemCall(offset byte, fn SystemCall) error {
	this.systemCalls[offset] = fn
	return nil
}
func (this *Core) SystemCall(inst *DecodedInstruction) error {
	return this.systemCalls[inst.Data[0]](this, inst)
}

func (this *Core) Dispatch(inst Instruction) error {
	this.advancePc = true
	if di, err := inst.Decode(); err != nil {
		return err
	} else {
		return this.Invoke(di)
	}
}
func terminateSystemCall(core *Core, inst *DecodedInstruction) error {
	core.terminateExecution = true
	return nil
}
func panicSystemCall(core *Core, inst *DecodedInstruction) error {
	// we don't want to panic the program itself but generate a new error
	// look at the data attached to the panic and encode it
	return NewError(ErrorPanic, uint(inst.Immediate()))
}
func defaultSystemCall(core *Core, inst *DecodedInstruction) error {
	return NewError(ErrorInvalidSystemCommand, uint(inst.Data[0]))
}

func (this *Core) ShouldExecute() bool {
	return this.terminateExecution
}
func (this *Core) HaltExecution() {
	this.terminateExecution = true
}
func (this *Core) ResumeExecution() {
	this.terminateExecution = false
}
func defaultExtendedUnit(core *Core, inst *DecodedInstruction) error {
	return NewError(ErrorInvalidInstructionGroupProvided, uint(inst.Group))
}

func (this *Core) InstructionAddress() Word {
	return this.Register(InstructionPointer)
}
func (this *Core) NextInstructionAddress() Word {
	return this.Register(InstructionPointer) + 1
}
func (this *Core) PredicateValue(index byte) bool {
	return this.Register(index) != 0
}

func NewDecodedInstruction(group, op, data0, data1, data2 byte) (*DecodedInstruction, error) {
	if group >= MajorOperationGroupCount {
		return nil, fmt.Errorf("Provided group (%d) is out of range!", group)
	} else {
		var di DecodedInstruction
		di.Group = group
		di.Op = op
		di.Data[0] = data0
		di.Data[1] = data1
		di.Data[2] = data2
		return &di, nil
	}
}
func NewDecodedInstructionImmediate(group, op, data0 byte, imm Word) (*DecodedInstruction, error) {
	return NewDecodedInstruction(group, op, data0, byte(imm), byte(imm>>8))
}

const (
	// System commands
	SystemCallTerminate = iota
	SystemCallPanic
	NumberOfSystemCalls
)

func init() {
	if NumberOfSystemCalls > 256 {
		panic("Too many system commands defined!")
	}
}

func (this *Core) TerminateExecution() bool {
	return this.terminateExecution
}

func (this *Core) CurrentInstruction() Instruction {
	return this.code[this.Register(InstructionPointer)]
}

func (this *Core) AdvanceProgramCounter() error {
	if this.advancePc {
		if err := this.SetRegister(InstructionPointer, this.NextInstructionAddress()); err != nil {
			return err
		}
	} else {
		this.advancePc = true
	}
	return nil
}

func (this *Core) ExecuteCurrentInstruction() error {
	return this.Dispatch(this.CurrentInstruction())
}
<commit_msg>Added encoding functions for iris1<commit_after>// machine description of iris1
package iris1

import (
	"fmt"
)

const (
	RegisterCount            = 256
	MemorySize               = 65536
	MajorOperationGroupCount = 8
	SystemCallCount          = 256
)
const (
	// reserved registers
	FalseRegister = iota
	TrueRegister
	InstructionPointer
	StackPointer
	PredicateRegister
	CountRegister
	CallPointer
	UserRegisterBegin
)

const (
	// Error codes
	ErrorNone = iota
	ErrorPanic
	ErrorGetRegisterOutOfRange
	ErrorPutRegisterOutOfRange
	ErrorInvalidInstructionGroupProvided
	ErrorInvalidArithmeticOperation
	ErrorInvalidMoveOperation
	ErrorInvalidJumpOperation
	ErrorInvalidCompareOperation
	ErrorInvalidMiscOperation
	ErrorInvalidSystemCommand
	ErrorWriteToFalseRegister
	ErrorWriteToTrueRegister
	ErrorEncodeByteOutOfRange
	ErrorGroupValueOutOfRange
	ErrorOpValueOutOfRange
)
const (
	// Instruction groups
	InstructionGroupArithmetic = iota
	InstructionGroupMove
	InstructionGroupJump
	InstructionGroupCompare
	InstructionGroupMisc
)

var errorLookup = []string{
	"None",
	"The core was sent a panic signal with argument %d!",
	"Attempted to get the value of invalid register r%d",
	"Attempted to set the value of invalid register r%d",
	"Instruction group %d is not a valid instruction group!",
	"Illegal arithmetic operation %d",
	"Illegal move operation %d",
	"Illegal jump operation %d",
	"Illegal compare operation %d",
	"Illegal misc operation %d",
	"Invalid system command %d",
	"Attempted to write %d to false register",
	"Attempted to write %d to true register!",
	"Specified illegal byte offset %d to encode data into",
	"Provided group id %d is larger than the space allotted to specifying the group",
	"Provided op id %d is larger than the space allotted to specifying the op",
}

type Word uint16
type Dword uint32
type Instruction Dword

func (this Instruction) group() byte {
	return byte(((this & 0x000000FF) & 0x7))
}
func (this Instruction) op() byte {
	return byte(((this & 0x000000FF) & 0xF8) >> 3)
}
func (this Instruction) register(index int) (byte, error) {
	switch index {
	case 0:
		return byte(this), nil
	case 1:
		return byte((this & 0x0000FF00) >> 8), nil
	case 2:
		return byte((this & 0x00FF0000) >> 16), nil
	case 3:
		return byte((this & 0xFF000000) >> 24), nil
	default:
		return 0, fmt.Errorf("Register index: %d is out of range!", index)
	}
}
func NewEncodedInstruction(group, op, dest, src0, src1 byte) *Instruction {
	// fuse group and op together
	var inst Instruction
	inst.setGroup(group)
	inst.setOp(op)
	_ = inst.setByte(1, dest)
	_ = inst.setByte(2, src0)
	_ = inst.setByte(3, src1)
	return &inst
}
func NewEncodedImmediateInstruction(group, op, dest byte, imm Word) *Instruction {
	var inst Instruction
	inst.setGroup(group)
	inst.setOp(op)
	_ = inst.setByte(1, dest)
	inst.setImmediate(imm)
	return &inst
}
func (this Instruction) immediate() Word {
	return Word((this & 0xFFFF0000) >> 16)
}

func (this *Instruction) setGroup(group byte) {
	*this = ((*this &^ 0x7) | Instruction(group))
}
func (this *Instruction) setOp(op byte) {
	*this = ((*this &^ 0xF8) | (Instruction(op) << 3))
}
func (this *Instruction) setByte(index int, value byte) error {
	switch index {
	case 1:
		*this = ((*this &^ 0x0000FF00) | (Instruction(value) << 8))
	case 2:
		*this = ((*this &^ 0x00FF0000) | (Instruction(value) << 16))
	case 3:
		*this = ((*this &^ 0xFF000000) | (Instruction(value) << 24))
	default:
		return NewError(ErrorEncodeByteOutOfRange, uint(index))
	}
	return nil
}
func (this *Instruction) setImmediate(value Word) {
	*this = ((*this &^ 0xFFFF0000) | (Instruction(value) << 16))
}

type DecodedInstruction struct {
	Group byte
	Op    byte
	Data  [3]byte
}

func (this Instruction) Decode() (*DecodedInstruction, error) {
	var di DecodedInstruction
	di.Group = this.group()
	di.Op = this.op()
	if value, err := this.register(1); err != nil {
		return nil, err
	} else {
		di.Data[0] = value
	}
	if value, err := this.register(2); err != nil {
		return nil, err
	} else {
		di.Data[1] = value
	}
	if value, err := this.register(3); err != nil {
		return nil, err
	} else {
		di.Data[2] = value
	}
	return &di, nil
}

func (this *DecodedInstruction) SetImmediate(value Word) {
	this.Data[1] = byte(value)
	this.Data[2] = byte(value >> 8)
}
func (this *DecodedInstruction) Immediate() Word {
	return Word((Word(this.Data[2]) << 8) | Word(this.Data[1]))
}

func (this *DecodedInstruction) Encode() *Instruction {
	i := new(Instruction)
	// encode group
	i.setGroup(this.Group)
	i.setOp(this.Op)
	i.setByte(1, this.Data[0])
	i.setByte(2, this.Data[1])
	i.setByte(3, this.Data[2])
	return i
}

type IrisError struct {
	value, code uint
}

func NewError(code, value uint) error {
	return &IrisError{code: code, value: value}
}

func (this IrisError) Error() string {
	if this.code == 0 {
		return fmt.Sprintf("No Error with value %d!!! This should never ever showup!", this.value)
	} else if this.code >= uint(len(errorLookup)) {
		return fmt.Sprintf("Unknown error %d with value %d! Something really bad happened!", this.code, this.value)
	} else {
		return fmt.Sprintf(errorLookup[this.code], this.value)
	}
}

type ExecutionUnit func(*Core, *DecodedInstruction) error
type SystemCall ExecutionUnit

type Core struct {
	gpr   [RegisterCount - UserRegisterBegin]Word
	code  [MemorySize]Instruction
	data  [MemorySize]Word
	stack [MemorySize]Word
	call  [MemorySize]Word
	// internal registers that should be easy to find
	instructionPointer Word
	stackPointer       Word
	callPointer        Word
	count              Word
	predicate          Word
	advancePc          bool
	terminateExecution bool
	groups             [MajorOperationGroupCount]ExecutionUnit
	systemCalls        [SystemCallCount]SystemCall
}

func (this *Core) SetRegister(index byte, value Word) error {
	switch index {
	case FalseRegister:
		return NewError(ErrorWriteToFalseRegister, uint(value))
	case TrueRegister:
		return NewError(ErrorWriteToTrueRegister, uint(value))
	case InstructionPointer:
		this.instructionPointer = value
	case StackPointer:
		this.stackPointer = value
	case PredicateRegister:
		this.predicate = value
	case CountRegister:
		this.count = value
	case CallPointer:
		this.callPointer = value
	default:
		this.gpr[index-UserRegisterBegin] = value
	}
	return nil
}
func (this *Core) Register(index byte) Word {
	switch index {
	case FalseRegister:
		return 0
	case TrueRegister:
		return 1
	case InstructionPointer:
		return this.instructionPointer
	case StackPointer:
		return this.stackPointer
	case PredicateRegister:
		return this.predicate
	case CountRegister:
		return this.count
	case CallPointer:
		return this.callPointer
	default:
		// do the offset calculation
		return this.gpr[index-UserRegisterBegin]
	}
}

func (this *Core) CodeMemory(address Word) Instruction {
	return this.code[address]
}
func (this *Core) SetCodeMemory(address Word, value Instruction) error {
	this.code[address] = value
	return nil
}
func (this *Core) Call(addr Word) error {
	this.callPointer++
	this.call[this.callPointer] = this.NextInstructionAddress()
	return this.SetRegister(InstructionPointer, addr)
}
func (this *Core) Return() Word {
	value := this.call[this.callPointer]
	this.callPointer--
	return value
}
func (this *Core) Push(value Word) {
	this.stackPointer++
	this.stack[this.stackPointer] = value
}
func (this *Core) Peek() Word {
	return this.stack[this.stackPointer]
}
func (this *Core) Pop() Word {
	value := this.stack[this.stackPointer]
	this.stackPointer--
	return value
}
func (this *Core) DataMemory(address Word) Word {
	return this.data[address]
}
func (this *Core) SetDataMemory(address, value Word) error {
	this.data[address] = value
	return nil
}

var defaultExecutionUnits = []struct {
	Group byte
	Unit  ExecutionUnit
}{
	{Group: InstructionGroupArithmetic, Unit: arithmetic},
	{Group: InstructionGroupMove, Unit: move},
	{Group: InstructionGroupJump, Unit: jump},
	{Group: InstructionGroupCompare, Unit: compare},
	{Group: InstructionGroupMisc, Unit: misc},
}

func New() (*Core, error) {
	var c Core
	c.advancePc = true
	c.terminateExecution = false
	if err := c.SetRegister(InstructionPointer, 0); err != nil {
		return nil, err
	}
	if err := c.SetRegister(PredicateRegister, 0); err != nil {
		return nil, err
	}
	if err := c.SetRegister(StackPointer, 0xFFFF); err != nil {
		return nil, err
	}
	if err := c.SetRegister(CallPointer, 0xFFFF); err != nil {
		return nil, err
	}
	if err := c.SetRegister(CountRegister, 0); err != nil {
		return nil, err
	}
	for i := 0; i < MajorOperationGroupCount; i++ {
		if err := c.InstallExecutionUnit(byte(i), defaultExtendedUnit); err != nil {
			return nil, err
		}
	}
	for _, unit := range defaultExecutionUnits {
		if err := c.InstallExecutionUnit(unit.Group, unit.Unit); err != nil {
			return nil, err
		}
	}
	for i := 0; i < SystemCallCount; i++ {
		if err := c.InstallSystemCall(byte(i), defaultSystemCall); err != nil {
			return nil, err
		}
	}
	c.InstallSystemCall(SystemCallTerminate, terminateSystemCall)
	c.InstallSystemCall(SystemCallPanic, panicSystemCall)
	return &c, nil
}

func (this *Core) InstallExecutionUnit(group byte, fn ExecutionUnit) error {
	if group >= MajorOperationGroupCount {
		return NewError(ErrorGroupValueOutOfRange, uint(group))
	} else {
		this.groups[group] = fn
		return nil
	}
}
func (this *Core) Invoke(inst *DecodedInstruction) error {
	return this.groups[inst.Group](this, inst)
}
func (this *Core) InstallSystemCall(offset byte, fn SystemCall) error {
	this.systemCalls[offset] = fn
	return nil
}
func (this *Core) SystemCall(inst *DecodedInstruction) error {
	return this.systemCalls[inst.Data[0]](this, inst)
}

func (this *Core) Dispatch(inst Instruction) error {
	this.advancePc = true
	if di, err := inst.Decode(); err != nil {
		return err
	} else {
		return this.Invoke(di)
	}
}
func terminateSystemCall(core *Core, inst *DecodedInstruction) error {
	core.terminateExecution = true
	return nil
}
func panicSystemCall(core *Core, inst *DecodedInstruction) error {
	// we don't want to panic the program itself but generate a new error
	// look at the data attached to the panic and encode it
	return NewError(ErrorPanic, uint(inst.Immediate()))
}
func defaultSystemCall(core *Core, inst *DecodedInstruction) error {
	return NewError(ErrorInvalidSystemCommand, uint(inst.Data[0]))
}

func (this *Core) ShouldExecute() bool {
	return this.terminateExecution
}
func (this *Core) HaltExecution() {
	this.terminateExecution = true
}
func (this *Core) ResumeExecution() {
	this.terminateExecution = false
}
func defaultExtendedUnit(core *Core, inst *DecodedInstruction) error {
	return NewError(ErrorInvalidInstructionGroupProvided, uint(inst.Group))
}

func (this *Core) InstructionAddress() Word {
	return this.Register(InstructionPointer)
}
func (this *Core) NextInstructionAddress() Word {
	return this.Register(InstructionPointer) + 1
}
func (this *Core) PredicateValue(index byte) bool {
	return this.Register(index) != 0
}

func NewDecodedInstruction(group, op, data0, data1, data2 byte) (*DecodedInstruction, error) {
	if group >= MajorOperationGroupCount {
		return nil, fmt.Errorf("Provided group (%d) is out of range!", group)
	} else {
		var di DecodedInstruction
		di.Group = group
		di.Op = op
		di.Data[0] = data0
		di.Data[1] = data1
		di.Data[2] = data2
		return &di, nil
	}
}
func NewDecodedInstructionImmediate(group, op, data0 byte, imm Word) (*DecodedInstruction, error) {
	return NewDecodedInstruction(group, op, data0, byte(imm), byte(imm>>8))
}

const (
	// System commands
	SystemCallTerminate = iota
	SystemCallPanic
	NumberOfSystemCalls
)

func init() {
	if NumberOfSystemCalls > 256 {
		panic("Too many system commands defined!")
	}
}

func (this *Core) TerminateExecution() bool {
	return this.terminateExecution
}

func (this *Core) CurrentInstruction() Instruction {
	return this.code[this.Register(InstructionPointer)]
}

func (this *Core) AdvanceProgramCounter() error {
	if this.advancePc {
		if err := this.SetRegister(InstructionPointer, this.NextInstructionAddress()); err != nil {
			return err
		}
	} else {
		this.advancePc = true
	}
	return nil
}

func (this *Core) ExecuteCurrentInstruction() error {
	return this.Dispatch(this.CurrentInstruction())
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"errors"

	"time"

	rgw "github.com/rook/rook/pkg/operator/ceph/object"
	"github.com/rook/rook/tests/framework/clients"
	"github.com/rook/rook/tests/framework/utils"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

var (
	userid           = "rook-user"
	userdisplayname  = "A rook RGW user"
	bucketname       = "smokebkt"
	objBody          = "Test Rook Object Data"
	objectKey        = "rookObj1"
	contentType      = "plain/text"
	storageClassName = "rook-smoke-delete-bucket"
	obcName          = "smoke-delete-bucket"
	region           = "us-east-1"
)

// Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order
// Create object store, Create User, Connect to Object Store, Create Bucket, Read/Write/Delete to bucket,
// Check issues in MGRs, Delete Bucket and Delete user
func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
	storeName := "teststore"
	defer objectTestDataCleanUp(helper, k8sh, namespace, storeName)

	logger.Infof("Object Storage End To End Integration Test - Create Object Store, User,Bucket and read/write to bucket")
	logger.Infof("Running on Rook Cluster %s", namespace)

	logger.Infof("Step 0 : Create Object Store User")
	cosuErr := helper.ObjectUserClient.Create(namespace, userid, userdisplayname, storeName)

	logger.Infof("Step 1 : Create Object Store")
	cobsErr := helper.ObjectClient.Create(namespace, storeName, 3)

	// check that ObjectStore is created
	require.Nil(s.T(), cobsErr)
	logger.Infof("Object store created successfully")

	// check that ObjectUser is created
	require.Nil(s.T(), cosuErr)
	logger.Infof("Waiting 10 seconds to ensure user was created")
	time.Sleep(10 * time.Second)
	logger.Infof("Checking to see if the user secret has been created")
	i := 0
	for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == false; i++ {
		logger.Infof("(%d) secret check sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	assert.True(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))
	userInfo, err := helper.ObjectUserClient.GetUser(namespace, storeName, userid)
	require.NoError(s.T(), err)
	assert.Equal(s.T(), userid, userInfo.UserID)
	assert.Equal(s.T(), userdisplayname, *userInfo.DisplayName)
	logger.Infof("Done creating object store user")

	logger.Infof("Step 2 : Test Deleting User")
	dosuErr := helper.ObjectUserClient.Delete(namespace, userid)
	require.Nil(s.T(), dosuErr)
	logger.Infof("Object store user deleted successfully")
	logger.Infof("Checking to see if the user secret has been deleted")
	i = 0
	for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == true; i++ {
		logger.Infof("(%d) secret check sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	assert.False(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))

	logger.Infof("Check that MGRs are not in a crashloop")
	assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", namespace, 1, "Running"))
	logger.Infof("Ceph MGRs are running")

	// Testing creation/deletion of objects using Object Bucket Claim
	logger.Infof("Step 3 : Create Object Bucket Claim with reclaim policy delete")
	cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, storageClassName, "Delete", region)
	require.Nil(s.T(), cobErr)
	cobcErr := helper.BucketClient.CreateObc(obcName, storageClassName, bucketname, true)
	require.Nil(s.T(), cobcErr)

	for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "created"); i++ {
		logger.Infof("(%d) obc created check, sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	require.NotEqual(s.T(), i, 4)

	logger.Infof("Check if bucket was created")
	context := k8sh.MakeContext()
	rgwcontext := rgw.NewContext(context, storeName, namespace)
	var bkt rgw.ObjectBucket
	for i = 0; i < 4; i++ {
		b, _, err := rgw.GetBucket(rgwcontext, bucketname)
		if b != nil && err == nil {
			bkt = *b
			break
		}
		logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	require.Equal(s.T(), bkt.Name, bucketname)
	logger.Infof("OBC, Secret and ConfigMap created")

	logger.Infof("Step 4 : Create s3 client")
	s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
	s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
	s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
	s3client := utils.CreateNewS3Helper(s3endpoint, s3AccessKey, s3SecretKey)
	logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)

	logger.Infof("Step 5 : Put Object on bucket")
	_, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType)
	require.Nil(s.T(), poErr)

	logger.Infof("Step 6 : Get Object from bucket")
	read, err := s3client.GetObjectInBucket(bucketname, objectKey)
	require.Nil(s.T(), err)
	require.Equal(s.T(), objBody, read)
	logger.Infof("Object Created and Retrieved on bucket successfully")

	logger.Infof("Step 7 : Delete object on bucket")
	_, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey)
	require.Nil(s.T(), delobjErr)
	logger.Infof("Object deleted on bucket successfully")

	logger.Infof("Step 8 : Delete Object Bucket Claim")
	dobcErr := helper.BucketClient.DeleteObc(obcName, storageClassName, bucketname, true)
	require.Nil(s.T(), dobcErr)
	logger.Infof("Checking to see if the obc, secret and cm have all been deleted")
	for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ {
		logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	assert.NotEqual(s.T(), i, 4)

	logger.Infof("ensure bucket was deleted")
	var rgwErr int
	for i = 0; i < 4; i++ {
		_, rgwErr, _ = rgw.GetBucket(rgwcontext, bucketname)
		if rgwErr == rgw.RGWErrorNotFound {
			break
		}
		logger.Infof("(%d) check bucket deleted, sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	assert.NotEqual(s.T(), i, 4)
	assert.Equal(s.T(), rgwErr, rgw.RGWErrorNotFound)

	dobErr := helper.BucketClient.DeleteBucketStorageClass(namespace, storeName, storageClassName, "Delete", region)
	assert.Nil(s.T(), dobErr)
	logger.Infof("Delete Object Bucket Claim successfully")

	// TODO : Add case for brownfield/cleanup s3 client

	logger.Infof("Delete Object Store")
	dobsErr := helper.ObjectClient.Delete(namespace, storeName)
	assert.Nil(s.T(), dobsErr)
	logger.Infof("Done deleting object store")
}

// Test Object StoreCreation on Rook that was installed via helm
func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, name string, replicaSize int) {
	logger.Infof("Object Storage End To End Integration Test - Create Object Store and check if rgw service is Running")
	logger.Infof("Running on Rook Cluster %s", namespace)

	logger.Infof("Step 1 : Create Object Store")
	err := helper.ObjectClient.Create(namespace, name, int32(replicaSize))
	require.Nil(s.T(), err)

	logger.Infof("Step 2 : check rook-ceph-rgw service status and count")
	require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", namespace, "Running"),
		"Make sure rook-ceph-rgw is in running state")

	assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"),
		"Make sure all rook-ceph-rgw pods are in Running state")

	require.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace))

}

func objectTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) {
	logger.Infof("FIX: Cleaning up object store")
	/*oc := helper.ObjectClient
	userinfo, err := helper.ObjectClient.ObjectGetUser(storeName, userid)
	if err != nil {
		return //when user is not found
	}
	s3endpoint, _ := k8sh.GetRGWServiceURL(storeName, namespace)
	s3client := utils.CreateNewS3Helper(s3endpoint, *userinfo.AccessKey, *userinfo.SecretKey)
	s3client.DeleteObjectInBucket(bucketname, objectKey)
	s3client.DeleteBucket(bucketname)
	helper.ObjectClient.DeleteUser(storeName, userid)*/
}

func getBucket(bucketname string, bucketList []rgw.ObjectBucket) (rgw.ObjectBucket, error) {
	for _, bucket := range bucketList {
		if bucket.Name == bucketname {
			return bucket, nil
		}
	}
	return rgw.ObjectBucket{}, errors.New("Bucket not found")
}

func getBucketSizeAndObjects(bucketname string, bucketList []rgw.ObjectBucket) (uint64, uint64, error) {
	bkt, err := getBucket(bucketname, bucketList)
	if err != nil {
		return 0, 0, errors.New("Bucket not found")
	}
	return bkt.Size, bkt.NumberOfObjects, nil
}
<commit_msg>Ceph: Create ObjectStore before ObjectStoreUser<commit_after>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"errors"

	"time"

	rgw "github.com/rook/rook/pkg/operator/ceph/object"
	"github.com/rook/rook/tests/framework/clients"
	"github.com/rook/rook/tests/framework/utils"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

var (
	userid           = "rook-user"
	userdisplayname  = "A rook RGW user"
	bucketname       = "smokebkt"
	objBody          = "Test Rook Object Data"
	objectKey        = "rookObj1"
	contentType      = "plain/text"
	storageClassName = "rook-smoke-delete-bucket"
	obcName          = "smoke-delete-bucket"
	region           = "us-east-1"
)

// Smoke Test for ObjectStore - Test check the following operations on ObjectStore in order
// Create object store, Create User, Connect to Object Store, Create Bucket, Read/Write/Delete to bucket,
// Check issues in MGRs, Delete Bucket and Delete user
func runObjectE2ETest(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string) {
	storeName := "teststore"
	defer objectTestDataCleanUp(helper, k8sh, namespace, storeName)

	logger.Infof("Object Storage End To End Integration Test - Create Object Store, User,Bucket and read/write to bucket")
	logger.Infof("Running on Rook Cluster %s", namespace)

	logger.Infof("Step 0 : Create Object Store")
	cobsErr := helper.ObjectClient.Create(namespace, storeName, 3)
	// check that ObjectStore is created
	require.Nil(s.T(), cobsErr)
	logger.Infof("Object store created successfully")

	logger.Infof("Step 1 : Create Object Store User")
	cosuErr := helper.ObjectUserClient.Create(namespace, userid, userdisplayname, storeName)
	// check that ObjectUser is created
	require.Nil(s.T(), cosuErr)
	logger.Infof("Waiting 10 seconds to ensure user was created")
	time.Sleep(10 * time.Second)
	logger.Infof("Checking to see if the user secret has been created")
	i := 0
	for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == false; i++ {
		logger.Infof("(%d) secret check sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	assert.True(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))
	userInfo, err := helper.ObjectUserClient.GetUser(namespace, storeName, userid)
	require.NoError(s.T(), err)
	assert.Equal(s.T(), userid, userInfo.UserID)
	assert.Equal(s.T(), userdisplayname, *userInfo.DisplayName)
	logger.Infof("Done creating object store user")

	logger.Infof("Step 2 : Test Deleting User")
	dosuErr := helper.ObjectUserClient.Delete(namespace, userid)
	require.Nil(s.T(), dosuErr)
	logger.Infof("Object store user deleted successfully")
	logger.Infof("Checking to see if the user secret has been deleted")
	i = 0
	for i = 0; i < 4 && helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid) == true; i++ {
		logger.Infof("(%d) secret check sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	assert.False(s.T(), helper.ObjectUserClient.UserSecretExists(namespace, storeName, userid))

	logger.Infof("Check that MGRs are not in a crashloop")
	assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-mgr", namespace, 1, "Running"))
	logger.Infof("Ceph MGRs are running")

	// Testing creation/deletion of objects using Object Bucket Claim
	logger.Infof("Step 3 : Create Object Bucket Claim with reclaim policy delete")
	cobErr := helper.BucketClient.CreateBucketStorageClass(namespace, storeName, storageClassName, "Delete", region)
	require.Nil(s.T(), cobErr)
	cobcErr := helper.BucketClient.CreateObc(obcName, storageClassName, bucketname, true)
	require.Nil(s.T(), cobcErr)

	for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "created"); i++ {
		logger.Infof("(%d) obc created check, sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	require.NotEqual(s.T(), i, 4)

	logger.Infof("Check if bucket was created")
	context := k8sh.MakeContext()
	rgwcontext := rgw.NewContext(context, storeName, namespace)
	var bkt rgw.ObjectBucket
	for i = 0; i < 4; i++ {
		b, _, err := rgw.GetBucket(rgwcontext, bucketname)
		if b != nil && err == nil {
			bkt = *b
			break
		}
		logger.Infof("(%d) check bucket exists, sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	require.Equal(s.T(), bkt.Name, bucketname)
	logger.Infof("OBC, Secret and ConfigMap created")

	logger.Infof("Step 4 : Create s3 client")
	s3endpoint, _ := helper.ObjectClient.GetEndPointUrl(namespace, storeName)
	s3AccessKey, _ := helper.BucketClient.GetAccessKey(obcName)
	s3SecretKey, _ := helper.BucketClient.GetSecretKey(obcName)
	s3client := utils.CreateNewS3Helper(s3endpoint, s3AccessKey, s3SecretKey)
	logger.Infof("endpoint (%s) Accesskey (%s) secret (%s)", s3endpoint, s3AccessKey, s3SecretKey)

	logger.Infof("Step 5 : Put Object on bucket")
	_, poErr := s3client.PutObjectInBucket(bucketname, objBody, objectKey, contentType)
	require.Nil(s.T(), poErr)

	logger.Infof("Step 6 : Get Object from bucket")
	read, err := s3client.GetObjectInBucket(bucketname, objectKey)
	require.Nil(s.T(), err)
	require.Equal(s.T(), objBody, read)
	logger.Infof("Object Created and Retrieved on bucket successfully")

	logger.Infof("Step 7 : Delete object on bucket")
	_, delobjErr := s3client.DeleteObjectInBucket(bucketname, objectKey)
	require.Nil(s.T(), delobjErr)
	logger.Infof("Object deleted on bucket successfully")

	logger.Infof("Step 8 : Delete Object Bucket Claim")
	dobcErr := helper.BucketClient.DeleteObc(obcName, storageClassName, bucketname, true)
	require.Nil(s.T(), dobcErr)
	logger.Infof("Checking to see if the obc, secret and cm have all been deleted")
	for i = 0; i < 4 && !helper.BucketClient.CheckOBC(obcName, "deleted"); i++ {
		logger.Infof("(%d) obc deleted check, sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	assert.NotEqual(s.T(), i, 4)

	logger.Infof("ensure bucket was deleted")
	var rgwErr int
	for i = 0; i < 4; i++ {
		_, rgwErr, _ = rgw.GetBucket(rgwcontext, bucketname)
		if rgwErr == rgw.RGWErrorNotFound {
			break
		}
		logger.Infof("(%d) check bucket deleted, sleeping for 5 seconds ...", i)
		time.Sleep(5 * time.Second)
	}
	assert.NotEqual(s.T(), i, 4)
	assert.Equal(s.T(), rgwErr, rgw.RGWErrorNotFound)

	dobErr := helper.BucketClient.DeleteBucketStorageClass(namespace, storeName, storageClassName, "Delete", region)
	assert.Nil(s.T(), dobErr)
	logger.Infof("Delete Object Bucket Claim successfully")

	// TODO : Add case for brownfield/cleanup s3 client

	logger.Infof("Delete Object Store")
	dobsErr := helper.ObjectClient.Delete(namespace, storeName)
	assert.Nil(s.T(), dobsErr)
	logger.Infof("Done deleting object store")
}

// Test Object StoreCreation on Rook that was installed via helm
func runObjectE2ETestLite(helper *clients.TestClient, k8sh *utils.K8sHelper, s suite.Suite, namespace string, name string, replicaSize int) {
	logger.Infof("Object Storage End To End Integration Test - Create Object Store and check if rgw service is Running")
	logger.Infof("Running on Rook Cluster %s", namespace)

	logger.Infof("Step 1 : Create Object Store")
	err := helper.ObjectClient.Create(namespace, name, int32(replicaSize))
	require.Nil(s.T(), err)

	logger.Infof("Step 2 : check rook-ceph-rgw service status and count")
	require.True(s.T(), k8sh.IsPodInExpectedState("rook-ceph-rgw", namespace, "Running"),
		"Make sure rook-ceph-rgw is in running state")

	assert.True(s.T(), k8sh.CheckPodCountAndState("rook-ceph-rgw", namespace, replicaSize, "Running"),
		"Make sure all rook-ceph-rgw pods are in Running state")

	require.True(s.T(), k8sh.IsServiceUp("rook-ceph-rgw-"+name, namespace))

}

func objectTestDataCleanUp(helper *clients.TestClient, k8sh *utils.K8sHelper, namespace, storeName string) {
	logger.Infof("FIX: Cleaning up object store")
	/*oc := helper.ObjectClient
	userinfo, err := helper.ObjectClient.ObjectGetUser(storeName, userid)
	if err != nil {
		return //when user is not found
	}
	s3endpoint, _ := k8sh.GetRGWServiceURL(storeName, namespace)
	s3client := utils.CreateNewS3Helper(s3endpoint, *userinfo.AccessKey, *userinfo.SecretKey)
	s3client.DeleteObjectInBucket(bucketname, objectKey)
	s3client.DeleteBucket(bucketname)
	helper.ObjectClient.DeleteUser(storeName, userid)*/
}

func getBucket(bucketname string, bucketList []rgw.ObjectBucket) (rgw.ObjectBucket, error) {
	for _, bucket := range bucketList {
		if bucket.Name == bucketname {
			return bucket, nil
		}
	}
	return rgw.ObjectBucket{}, errors.New("Bucket not found")
}

func getBucketSizeAndObjects(bucketname string, bucketList []rgw.ObjectBucket) (uint64, uint64, error) {
	bkt, err := getBucket(bucketname, bucketList)
	if err != nil {
		return 0, 0, errors.New("Bucket not found")
	}
	return bkt.Size, bkt.NumberOfObjects, nil
}
<|endoftext|>


<commit_before>package get

import (
	"compress/gzip"
	"crypto"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"path"

	"github.com/moio/minima/util"
)

// common

// XMLLocation maps a <location> tag in repodata/repomd.xml or repodata/<ID>-primary.xml.gz
type XMLLocation struct {
	Href string `xml:"href,attr"`
}

// repodata/repomd.xml

// XMLRepomd maps a <repomd> tag in repodata/repomd.xml
type XMLRepomd struct {
	Data []XMLData `xml:"data"`
}

// XMLData maps a <data> tag in repodata/repomd.xml
type XMLData struct {
	Type     string      `xml:"type,attr"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// repodata/<ID>-primary.xml.gz

// XMLMetaData maps a <metadata> tag in repodata/<ID>-primary.xml.gz
type XMLMetaData struct {
	Packages []XMLPackage `xml:"package"`
}

// XMLPackage maps a <package> tag in repodata/<ID>-primary.xml.gz
type XMLPackage struct {
	Arch     string      `xml:"arch"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// XMLChecksum maps a <checksum> tag in repodata/<ID>-primary.xml.gz
type XMLChecksum struct {
	Type     string `xml:"type,attr"`
	Checksum string `xml:",cdata"`
}

var hashMap = map[string]crypto.Hash{
	"sha":    crypto.SHA1,
	"sha1":   crypto.SHA1,
	"sha256": crypto.SHA256,
}

const repomdPath = "repodata/repomd.xml"

// Syncer syncs repos from an HTTP source to a Storage
type Syncer struct {
	// URL of the repo this syncer syncs
	Url     string
	archs   map[string]bool
	storage Storage
}

// NewSyncer creates a new Syncer
func NewSyncer(url string, archs map[string]bool, storage Storage) *Syncer {
	return &Syncer{url, archs, storage}
}

// StoreRepo stores an HTTP repo in a Storage, automatically retrying in case of recoverable errors
func (r *Syncer) StoreRepo() (err error) {
	checksumMap := r.readChecksumMap()
	for i := 0; i < 20; i++ {
		err = r.storeRepo(checksumMap)
		if err == nil {
			return
		}

		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode {
			if uerr.StatusCode == 404 {
				log.Printf("Got 404, presumably temporarily, retrying...\n")
			} else {
				return err
			}
		}

		_, checksumError := err.(*util.ChecksumError)
		if checksumError {
			log.Printf(err.Error())
			log.Printf("Checksum did not match, presumably the repo was published while syncing, retrying...\n")
		} else {
			return err
		}
	}

	log.Printf("Too many temporary errors, aborting...\n")
	return err
}

// StoreRepo stores an HTTP repo in a Storage
func (r *Syncer) storeRepo(checksumMap map[string]XMLChecksum) (err error) {
	packagesToDownload, packagesToRecycle, err := r.processMetadata(checksumMap)
	if err != nil {
		return
	}

	downloadCount := len(packagesToDownload)
	log.Printf("Downloading %v packages...\n", downloadCount)
	for i, pack := range packagesToDownload {
		description := fmt.Sprintf("(%v/%v) %v", i+1, downloadCount, path.Base(pack.Location.Href))
		err = r.downloadStoreApply(pack.Location.Href, pack.Checksum.Checksum, description, hashMap[pack.Checksum.Type], util.Nop)
		if err != nil {
			return err
		}
	}

	recycleCount := len(packagesToRecycle)
	log.Printf("Recycling %v packages...\n", recycleCount)
	for _, pack := range packagesToRecycle {
		err = r.storage.Recycle(pack.Location.Href)
		if err != nil {
			return
		}
	}

	log.Printf("Committing changes...\n")
	err = r.storage.Commit()
	if err != nil {
		return
	}
	return
}

// downloadStore downloads a repo-relative path into a file
func (r *Syncer) downloadStore(path string, description string) error {
	return r.downloadStoreApply(path, "", description, 0, util.Nop)
}

// downloadStoreApply downloads a repo-relative path into a file, while applying a ReaderConsumer
func (r *Syncer) downloadStoreApply(path string, checksum string, description string, hash crypto.Hash, f util.ReaderConsumer) error {
	log.Printf("Downloading %v...", description)
	body, err := ReadURL(r.Url + "/" + path)
	if err != nil {
		return err
	}

	return util.Compose(r.storage.StoringMapper(path, checksum, hash), f)(body)
}

// processMetadata stores the repo metadata and returns a list of package file
// paths to download
func (r *Syncer) processMetadata(checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	err = r.downloadStoreApply(repomdPath, "", path.Base(repomdPath), 0, func(reader io.ReadCloser) (err error) {
		decoder := xml.NewDecoder(reader)
		var repomd XMLRepomd
		err = decoder.Decode(&repomd)
		if err != nil {
			return
		}

		data := repomd.Data
		for i := 0; i < len(data); i++ {
			metadataLocation := data[i].Location.Href
			metadataChecksum := data[i].Checksum
			decision := r.decide(metadataLocation, metadataChecksum, checksumMap)
			switch decision {
			case Download:
				err = r.downloadStore(metadataLocation, path.Base(metadataLocation))
				if err != nil {
					return
				}
			case Recycle:
				r.storage.Recycle(metadataLocation)
			}

			if data[i].Type == "primary" {
				packagesToDownload, packagesToRecycle, err = r.processPrimary(metadataLocation, checksumMap)
			}
		}
		return
	})
	if err != nil {
		return
	}

	ascPath := repomdPath + ".asc"
	err = r.downloadStore(ascPath, path.Base(ascPath))
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
		} else {
			return
		}
	}

	keyPath := repomdPath + ".key"
	err = r.downloadStore(keyPath, path.Base(keyPath))
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
			err = nil
		} else {
			return
		}
	}

	return
}

func (r *Syncer) readMetaData(reader io.Reader) (primary XMLMetaData, err error) {
	gzReader, err := gzip.NewReader(reader)
	if err != nil {
		return
	}
	defer gzReader.Close()

	decoder := xml.NewDecoder(gzReader)
	err = decoder.Decode(&primary)

	return
}

func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) {
	checksumMap = make(map[string]XMLChecksum)
	repomdReader, err := r.storage.NewReader(repomdPath, Permanent)
	if err != nil {
		if err == ErrFileNotFound {
			log.Println("First-time sync started")
		} else {
			log.Println(err.Error())
			log.Println("Error while reading previously-downloaded metadata. Starting sync from scratch")
		}
		return
	}
	defer repomdReader.Close()

	decoder := xml.NewDecoder(repomdReader)
	var repomd XMLRepomd
	err = decoder.Decode(&repomd)
	if err != nil {
		log.Println(err.Error())
		log.Println("Error while parsing previously-downloaded metadata. Starting sync from scratch")
		return
	}

	data := repomd.Data
	for i := 0; i < len(data); i++ {
		dataHref := data[i].Location.Href
		dataChecksum := data[i].Checksum
		checksumMap[dataHref] = dataChecksum
		if data[i].Type == "primary" {
			primaryReader, err := r.storage.NewReader(dataHref, Permanent)
			if err != nil {
				return
			}
			primary, err := r.readMetaData(primaryReader)
			if err != nil {
				return
			}
			for _, pack := range primary.Packages {
				checksumMap[pack.Location.Href] = pack.Checksum
			}
		}
	}
	return
}

// processPrimary stores the primary XML metadata file and returns a list of
// package file paths to download
func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	reader, err := r.storage.NewReader(path, Temporary)
	if err != nil {
		return
	}
	primary, err := r.readMetaData(reader)
	if err != nil {
		return
	}

	allArchs := len(r.archs) == 0
	for _, pack := range primary.Packages {
		if allArchs || pack.Arch == "noarch" || r.archs[pack.Arch] {
			decision := r.decide(pack.Location.Href, pack.Checksum, checksumMap)
			switch decision {
			case Download:
				packagesToDownload = append(packagesToDownload, pack)
			case Recycle:
				packagesToRecycle = append(packagesToRecycle, pack)
			}
		}
	}
	return
}

// Decision encodes what to do with a file
type Decision int

const (
	// Download means the Syncer will download a file
	Download Decision = iota
	// Recycle means the Syncer will copy an existing file without downloading
	Recycle
	// Skip means the Syncer detected an already-existing file and has nothing to do
	Skip
)

func (r *Syncer) decide(location string, checksum XMLChecksum, checksumMap map[string]XMLChecksum) Decision {
	previousChecksum, foundInPermanentLocation := checksumMap[location]
	if !foundInPermanentLocation || previousChecksum.Type != checksum.Type || previousChecksum.Checksum != checksum.Checksum {
		reader, err := r.storage.NewReader(location, Temporary)
		if err != nil {
			return Download
		}
		defer reader.Close()
		readChecksum, err := util.Checksum(reader, hashMap[checksum.Type])
		if err != nil || readChecksum != checksum.Checksum {
			return Download
		}
		return Skip
	}
	return Recycle
}
<commit_msg>Bugfix: check metadata checksums properly<commit_after>package get

import (
	"compress/gzip"
	"crypto"
	"encoding/xml"
	"fmt"
	"io"
	"log"
	"path"

	"github.com/moio/minima/util"
)

// common

// XMLLocation maps a <location> tag in repodata/repomd.xml or repodata/<ID>-primary.xml.gz
type XMLLocation struct {
	Href string `xml:"href,attr"`
}

// repodata/repomd.xml

// XMLRepomd maps a <repomd> tag in repodata/repomd.xml
type XMLRepomd struct {
	Data []XMLData `xml:"data"`
}

// XMLData maps a <data> tag in repodata/repomd.xml
type XMLData struct {
	Type     string      `xml:"type,attr"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// repodata/<ID>-primary.xml.gz

// XMLMetaData maps a <metadata> tag in repodata/<ID>-primary.xml.gz
type XMLMetaData struct {
	Packages []XMLPackage `xml:"package"`
}

// XMLPackage maps a <package> tag in repodata/<ID>-primary.xml.gz
type XMLPackage struct {
	Arch     string      `xml:"arch"`
	Location XMLLocation `xml:"location"`
	Checksum XMLChecksum `xml:"checksum"`
}

// XMLChecksum maps a <checksum> tag in repodata/<ID>-primary.xml.gz
type XMLChecksum struct {
	Type     string `xml:"type,attr"`
	Checksum string `xml:",cdata"`
}

var hashMap = map[string]crypto.Hash{
	"sha":    crypto.SHA1,
	"sha1":   crypto.SHA1,
	"sha256": crypto.SHA256,
}

const repomdPath = "repodata/repomd.xml"

// Syncer syncs repos from an HTTP source to a Storage
type Syncer struct {
	// URL of the repo this syncer syncs
	Url     string
	archs   map[string]bool
	storage Storage
}

// NewSyncer creates a new Syncer
func NewSyncer(url string, archs map[string]bool, storage Storage) *Syncer {
	return &Syncer{url, archs, storage}
}

// StoreRepo stores an HTTP repo in a Storage, automatically retrying in case of recoverable errors
func (r *Syncer) StoreRepo() (err error) {
	checksumMap := r.readChecksumMap()
	for i := 0; i < 20; i++ {
		err = r.storeRepo(checksumMap)
		if err == nil {
			return
		}

		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode {
			if uerr.StatusCode == 404 {
				log.Printf("Got 404, presumably temporarily, retrying...\n")
			} else {
				return err
			}
		}

		_, checksumError := err.(*util.ChecksumError)
		if checksumError {
			log.Printf(err.Error())
			log.Printf("Checksum did not match, presumably the repo was published while syncing, retrying...\n")
		} else {
			return err
		}
	}

	log.Printf("Too many temporary errors, aborting...\n")
	return err
}

// StoreRepo stores an HTTP repo in a Storage
func (r *Syncer) storeRepo(checksumMap map[string]XMLChecksum) (err error) {
	packagesToDownload, packagesToRecycle, err := r.processMetadata(checksumMap)
	if err != nil {
		return
	}

	downloadCount := len(packagesToDownload)
	log.Printf("Downloading %v packages...\n", downloadCount)
	for i, pack := range packagesToDownload {
		description := fmt.Sprintf("(%v/%v) %v", i+1, downloadCount, path.Base(pack.Location.Href))
		err = r.downloadStoreApply(pack.Location.Href, pack.Checksum.Checksum, description, hashMap[pack.Checksum.Type], util.Nop)
		if err != nil {
			return err
		}
	}

	recycleCount := len(packagesToRecycle)
	log.Printf("Recycling %v packages...\n", recycleCount)
	for _, pack := range packagesToRecycle {
		err = r.storage.Recycle(pack.Location.Href)
		if err != nil {
			return
		}
	}

	log.Printf("Committing changes...\n")
	err = r.storage.Commit()
	if err != nil {
		return
	}
	return
}

// downloadStore downloads a repo-relative path into a file
func (r *Syncer) downloadStore(path string, description string) error {
	return r.downloadStoreApply(path, "", description, 0, util.Nop)
}

// downloadStoreApply downloads a repo-relative path into a file, while applying a ReaderConsumer
func (r *Syncer) downloadStoreApply(path string, checksum string, description string, hash crypto.Hash, f util.ReaderConsumer) error {
	log.Printf("Downloading %v...", description)
	body, err := ReadURL(r.Url + "/" + path)
	if err != nil {
		return err
	}

	return util.Compose(r.storage.StoringMapper(path, checksum, hash), f)(body)
}

// processMetadata stores the repo metadata and returns a list of package file
// paths to download
func (r *Syncer) processMetadata(checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	err = r.downloadStoreApply(repomdPath, "", path.Base(repomdPath), 0, func(reader io.ReadCloser) (err error) {
		decoder := xml.NewDecoder(reader)
		var repomd XMLRepomd
		err = decoder.Decode(&repomd)
		if err != nil {
			return
		}

		data := repomd.Data
		for i := 0; i < len(data); i++ {
			metadataLocation := data[i].Location.Href
			metadataChecksum := data[i].Checksum
			decision := r.decide(metadataLocation, metadataChecksum, checksumMap)
			switch decision {
			case Download:
				err = r.downloadStoreApply(metadataLocation, metadataChecksum.Checksum, path.Base(metadataLocation), hashMap[metadataChecksum.Type], util.Nop)
				if err != nil {
					return
				}
			case Recycle:
				r.storage.Recycle(metadataLocation)
			}

			if data[i].Type == "primary" {
				packagesToDownload, packagesToRecycle, err = r.processPrimary(metadataLocation, checksumMap)
			}
		}
		return
	})
	if err != nil {
		return
	}

	ascPath := repomdPath + ".asc"
	err = r.downloadStore(ascPath, path.Base(ascPath))
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
		} else {
			return
		}
	}

	keyPath := repomdPath + ".key"
	err = r.downloadStore(keyPath, path.Base(keyPath))
	if err != nil {
		uerr, unexpectedStatusCode := err.(*UnexpectedStatusCodeError)
		if unexpectedStatusCode && uerr.StatusCode == 404 {
			log.Printf("Got 404, ignoring...")
			err = nil
		} else {
			return
		}
	}

	return
}

func (r *Syncer) readMetaData(reader io.Reader) (primary XMLMetaData, err error) {
	gzReader, err := gzip.NewReader(reader)
	if err != nil {
		return
	}
	defer gzReader.Close()

	decoder := xml.NewDecoder(gzReader)
	err = decoder.Decode(&primary)

	return
}

func (r *Syncer) readChecksumMap() (checksumMap map[string]XMLChecksum) {
	checksumMap = make(map[string]XMLChecksum)
	repomdReader, err := r.storage.NewReader(repomdPath, Permanent)
	if err != nil {
		if err == ErrFileNotFound {
			log.Println("First-time sync started")
		} else {
			log.Println(err.Error())
			log.Println("Error while reading previously-downloaded metadata. Starting sync from scratch")
		}
		return
	}
	defer repomdReader.Close()

	decoder := xml.NewDecoder(repomdReader)
	var repomd XMLRepomd
	err = decoder.Decode(&repomd)
	if err != nil {
		log.Println(err.Error())
		log.Println("Error while parsing previously-downloaded metadata. Starting sync from scratch")
		return
	}

	data := repomd.Data
	for i := 0; i < len(data); i++ {
		dataHref := data[i].Location.Href
		dataChecksum := data[i].Checksum
		checksumMap[dataHref] = dataChecksum
		if data[i].Type == "primary" {
			primaryReader, err := r.storage.NewReader(dataHref, Permanent)
			if err != nil {
				return
			}
			primary, err := r.readMetaData(primaryReader)
			if err != nil {
				return
			}
			for _, pack := range primary.Packages {
				checksumMap[pack.Location.Href] = pack.Checksum
			}
		}
	}
	return
}

// processPrimary stores the primary XML metadata file and returns a list of
// package file paths to download
func (r *Syncer) processPrimary(path string, checksumMap map[string]XMLChecksum) (packagesToDownload []XMLPackage, packagesToRecycle []XMLPackage, err error) {
	reader, err := r.storage.NewReader(path, Temporary)
	if err != nil {
		return
	}
	primary, err := r.readMetaData(reader)
	if err != nil {
		return
	}

	allArchs := len(r.archs) == 0
	for _, pack := range primary.Packages {
		if allArchs || pack.Arch == "noarch" || r.archs[pack.Arch] {
			decision := r.decide(pack.Location.Href, pack.Checksum, checksumMap)
			switch decision {
			case Download:
				packagesToDownload = append(packagesToDownload, pack)
			case Recycle:
				packagesToRecycle = append(packagesToRecycle, pack)
			}
		}
	}
	return
}

// Decision encodes what to do with a file
type Decision int

const (
	// Download means the Syncer will download a file
	Download Decision = iota
	// Recycle means the Syncer will copy an existing file without downloading
	Recycle
	// Skip means the Syncer detected an already-existing file and has nothing to do
	Skip
)

func (r *Syncer) decide(location string, checksum XMLChecksum, checksumMap map[string]XMLChecksum) Decision {
	previousChecksum, foundInPermanentLocation := checksumMap[location]
	if !foundInPermanentLocation || previousChecksum.Type != checksum.Type || previousChecksum.Checksum != checksum.Checksum {
		reader, err := r.storage.NewReader(location, Temporary)
		if err != nil {
			return Download
		}
		defer reader.Close()
		readChecksum, err := util.Checksum(reader, hashMap[checksum.Type])
		if err != nil || readChecksum != checksum.Checksum {
			return Download
		}
		return Skip
	}
	return Recycle
}
<|endoftext|>