<commit_before>package network

import (
	"fmt"
	"net"
	"strconv"

	"github.com/pions/pkg/stun"
	"github.com/pions/webrtc/internal/dtls"
	"github.com/pions/webrtc/internal/srtp"
	"github.com/pions/webrtc/pkg/rtp"
	"github.com/pkg/errors"
	"golang.org/x/net/ipv4"
)

func packetHandler(conn *ipv4.PacketConn, srcString string, remoteKey []byte, tlscfg *dtls.TLSCfg, b BufferTransportGenerator) {
	const MTU = 8192
	buffer := make([]byte, MTU)

	dtlsStates := make(map[string]*dtls.State)
	bufferTransports := make(map[uint32]chan<- *rtp.Packet)

	// TODO multiple SRTP Contexts
	// https://tools.ietf.org/html/rfc3711#section-3.2.3
	// A cryptographic context SHALL be uniquely identified by the triplet
	//  <SSRC, destination network address, destination transport port number>
	var srtpContext *srtp.Context
	for {
		n, _, rawDstAddr, err := conn.ReadFrom(buffer)
		if err != nil {
			fmt.Printf("Failed to read packet: %s \n", err.Error())
			continue
		}
		d, haveHandshaked := dtlsStates[rawDstAddr.String()]

		if haveHandshaked {
			if handled, certPair := d.MaybeHandleDTLSPacket(buffer, n); handled {
				if certPair != nil {
					srtpContext, err = srtp.CreateContext([]byte(certPair.ServerWriteKey[0:16]), []byte(certPair.ServerWriteKey[16:]), certPair.Profile)
					if err != nil {
						fmt.Println(err)
						continue
					}
				}
				continue
			}
		}

		if packetType, err := stun.GetPacketType(buffer[:n]); err == nil && packetType == stun.PacketTypeSTUN {
			if m, err := stun.NewMessage(buffer[:n]); err == nil && m.Class == stun.ClassRequest && m.Method == stun.MethodBinding {
				dstAddr := &stun.TransportAddr{IP: rawDstAddr.(*net.UDPAddr).IP, Port: rawDstAddr.(*net.UDPAddr).Port}
				if err := stun.BuildAndSend(conn, dstAddr, stun.ClassSuccessResponse, stun.MethodBinding, m.TransactionID,
					&stun.XorMappedAddress{
						XorAddress: stun.XorAddress{
							IP:   dstAddr.IP,
							Port: dstAddr.Port,
						},
					},
					&stun.MessageIntegrity{
						Key: remoteKey,
					},
					&stun.Fingerprint{},
				); err != nil {
					fmt.Println(err)
				}
			}
		} else if srtpContext != nil {
			// Make copy of packet
			// buffer[:n] can't be modified outside of network loop
			rawPacket := make([]byte, n)
			copy(rawPacket, buffer[:n])

			packet := &rtp.Packet{}
			if err := packet.Unmarshal(rawPacket); err != nil {
				fmt.Println("Failed to unmarshal RTP packet")
				continue
			}

			if ok := srtpContext.DecryptPacket(packet); !ok {
				fmt.Println("Failed to decrypt packet")
				continue
			}

			bufferTransport := bufferTransports[packet.SSRC]
			if bufferTransport == nil {
				bufferTransport = b(packet.SSRC)
				if bufferTransport == nil {
					fmt.Println("Failed to generate buffer transport, onTrack should be defined")
					continue
				}
				bufferTransports[packet.SSRC] = bufferTransport
			}
			bufferTransport <- packet
		} else {
			fmt.Println("SRTP packet, but no srtpSession")
		}

		if !haveHandshaked {
			d, err := dtls.NewState(tlscfg, true, srcString, rawDstAddr.String())
			if err != nil {
				fmt.Println(err)
				continue
			}

			d.DoHandshake()
			dtlsStates[rawDstAddr.String()] = d
		}
	}
}

// BufferTransportGenerator generates a new channel for the associated SSRC
// This channel is used to send RTP packets to users of pion-WebRTC
type BufferTransportGenerator func(uint32) chan<- *rtp.Packet

// UDPListener starts a new UDPListener, and will send the DTLS handshake if we are the client
func UDPListener(ip string, remoteKey []byte, tlscfg *dtls.TLSCfg, b BufferTransportGenerator) (int, error) {
	listener, err := net.ListenPacket("udp4", ip+":0")
	if err != nil {
		return 0, errors.Wrap(err, "Failed to listen packet")
	}

	addr, err := stun.NewTransportAddr(listener.LocalAddr())

	srcString := ip + ":" + strconv.Itoa(addr.Port)

	dtls.AddListener(srcString, conn)
	go packetHandler(conn, srcString, remoteKey, tlscfg, b)
	return addr.Port, err
}
<commit_msg>Add conn back<commit_after>package network

import (
	"fmt"
	"net"
	"strconv"

	"github.com/pions/pkg/stun"
	"github.com/pions/webrtc/internal/dtls"
	"github.com/pions/webrtc/internal/srtp"
	"github.com/pions/webrtc/pkg/rtp"
	"github.com/pkg/errors"
	"golang.org/x/net/ipv4"
)

func packetHandler(conn *ipv4.PacketConn, srcString string, remoteKey []byte, tlscfg *dtls.TLSCfg, b BufferTransportGenerator) {
	const MTU = 8192
	buffer := make([]byte, MTU)

	dtlsStates := make(map[string]*dtls.State)
	bufferTransports := make(map[uint32]chan<- *rtp.Packet)

	// TODO multiple SRTP Contexts
	// https://tools.ietf.org/html/rfc3711#section-3.2.3
	// A cryptographic context SHALL be uniquely identified by the triplet
	//  <SSRC, destination network address, destination transport port number>
	var srtpContext *srtp.Context
	for {
		n, _, rawDstAddr, err := conn.ReadFrom(buffer)
		if err != nil {
			fmt.Printf("Failed to read packet: %s \n", err.Error())
			continue
		}
		d, haveHandshaked := dtlsStates[rawDstAddr.String()]

		if haveHandshaked {
			if handled, certPair := d.MaybeHandleDTLSPacket(buffer, n); handled {
				if certPair != nil {
					srtpContext, err = srtp.CreateContext([]byte(certPair.ServerWriteKey[0:16]), []byte(certPair.ServerWriteKey[16:]), certPair.Profile)
					if err != nil {
						fmt.Println(err)
						continue
					}
				}
				continue
			}
		}

		if packetType, err := stun.GetPacketType(buffer[:n]); err == nil && packetType == stun.PacketTypeSTUN {
			if m, err := stun.NewMessage(buffer[:n]); err == nil && m.Class == stun.ClassRequest && m.Method == stun.MethodBinding {
				dstAddr := &stun.TransportAddr{IP: rawDstAddr.(*net.UDPAddr).IP, Port: rawDstAddr.(*net.UDPAddr).Port}
				if err := stun.BuildAndSend(conn, dstAddr, stun.ClassSuccessResponse, stun.MethodBinding, m.TransactionID,
					&stun.XorMappedAddress{
						XorAddress: stun.XorAddress{
							IP:   dstAddr.IP,
							Port: dstAddr.Port,
						},
					},
					&stun.MessageIntegrity{
						Key: remoteKey,
					},
					&stun.Fingerprint{},
				); err != nil {
					fmt.Println(err)
				}
			}
		} else if srtpContext != nil {
			// Make copy of packet
			// buffer[:n] can't be modified outside of network loop
			rawPacket := make([]byte, n)
			copy(rawPacket, buffer[:n])

			packet := &rtp.Packet{}
			if err := packet.Unmarshal(rawPacket); err != nil {
				fmt.Println("Failed to unmarshal RTP packet")
				continue
			}

			if ok := srtpContext.DecryptPacket(packet); !ok {
				fmt.Println("Failed to decrypt packet")
				continue
			}

			bufferTransport := bufferTransports[packet.SSRC]
			if bufferTransport == nil {
				bufferTransport = b(packet.SSRC)
				if bufferTransport == nil {
					fmt.Println("Failed to generate buffer transport, onTrack should be defined")
					continue
				}
				bufferTransports[packet.SSRC] = bufferTransport
			}
			bufferTransport <- packet
		} else {
			fmt.Println("SRTP packet, but no srtpSession")
		}

		if !haveHandshaked {
			d, err := dtls.NewState(tlscfg, true, srcString, rawDstAddr.String())
			if err != nil {
				fmt.Println(err)
				continue
			}

			d.DoHandshake()
			dtlsStates[rawDstAddr.String()] = d
		}
	}
}

// BufferTransportGenerator generates a new channel for the associated SSRC
// This channel is used to send RTP packets to users of pion-WebRTC
type BufferTransportGenerator func(uint32) chan<- *rtp.Packet

// UDPListener starts a new UDPListener, and will send the DTLS handshake if we are the client
func UDPListener(ip string, remoteKey []byte, tlscfg *dtls.TLSCfg, b BufferTransportGenerator) (int, error) {
	listener, err := net.ListenPacket("udp4", ip+":0")
	if err != nil {
		return 0, errors.Wrap(err, "Failed to listen packet")
	}

	conn := ipv4.NewPacketConn(listener)

	addr, err := stun.NewTransportAddr(listener.LocalAddr())

	srcString := ip + ":" + strconv.Itoa(addr.Port)

	dtls.AddListener(srcString, conn)
	go packetHandler(conn, srcString, remoteKey, tlscfg, b)
	return addr.Port, err
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"fmt"
	"net"
	"regexp"
	"strconv"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// A Server is an in-process gRPC server, listening on a system-chosen port on
// the local loopback interface. Servers are for testing only and are not
// intended to be used in production code.
//
// To create a server, make a new Server, register your handlers, then call
// Start:
//
//	srv, err := NewServer()
//	...
//	mypb.RegisterMyServiceServer(srv.Gsrv, &myHandler)
//	....
//	srv.Start()
//
// Clients should connect to the server with no security:
//
//	conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
//	...
type Server struct {
	Addr string
	Port int
	l    net.Listener
	Gsrv *grpc.Server
}

// NewServer creates a new Server. The Server will be listening for gRPC connections
// at the address named by the Addr field, without TLS.
func NewServer(opts ...grpc.ServerOption) (*Server, error) {
	return NewServerWithPort(0, opts...)
}

// NewServerWithPort creates a new Server at a specific port. The Server will be listening
// for gRPC connections at the address named by the Addr field, without TLS.
func NewServerWithPort(port int, opts ...grpc.ServerOption) (*Server, error) {
	l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
	if err != nil {
		return nil, err
	}
	s := &Server{
		Addr: l.Addr().String(),
		Port: parsePort(l.Addr().String()),
		l:    l,
		Gsrv: grpc.NewServer(opts...),
	}
	return s, nil
}

// Start causes the server to start accepting incoming connections.
// Call Start after registering handlers.
func (s *Server) Start() {
	go func() {
		if err := s.Gsrv.Serve(s.l); err != nil {
			panic(err)
		}
	}()
}

// Close shuts down the server.
func (s *Server) Close() {
	s.Gsrv.Stop()
	s.l.Close()
}

// PageBounds converts an incoming page size and token from an RPC request into
// slice bounds and the outgoing next-page token.
//
// PageBounds assumes that the complete, unpaginated list of items exists as a
// single slice. In addition to the page size and token, PageBounds needs the
// length of that slice.
//
// PageBounds's first two return values should be used to construct a sub-slice of
// the complete, unpaginated slice. E.g. if the complete slice is s, then
// s[from:to] is the desired page. Its third return value should be set as the
// NextPageToken field of the RPC response.
func PageBounds(pageSize int, pageToken string, length int) (from, to int, nextPageToken string, err error) {
	from, to = 0, length
	if pageToken != "" {
		from, err = strconv.Atoi(pageToken)
		if err != nil {
			return 0, 0, "", status.Errorf(codes.InvalidArgument, "bad page token: %v", err)
		}
		if from >= length {
			return length, length, "", nil
		}
	}
	if pageSize > 0 && from+pageSize < length {
		to = from + pageSize
		nextPageToken = strconv.Itoa(to)
	}
	return from, to, nextPageToken, nil
}

var portParser = regexp.MustCompile(`:[0-9]+`)

func parsePort(addr string) int {
	res := portParser.FindAllString(addr, -1)
	if len(res) == 0 {
		panic(fmt.Errorf("parsePort: found no numbers in %s", addr))
	}
	stringPort := res[0][1:] // strip the :
	p, err := strconv.ParseInt(stringPort, 10, 32)
	if err != nil {
		panic(err)
	}
	return int(p)
}
<commit_msg>testutil: change panic to log<commit_after>/*
Copyright 2016 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"fmt"
	"log"
	"net"
	"regexp"
	"strconv"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// A Server is an in-process gRPC server, listening on a system-chosen port on
// the local loopback interface. Servers are for testing only and are not
// intended to be used in production code.
//
// To create a server, make a new Server, register your handlers, then call
// Start:
//
//	srv, err := NewServer()
//	...
//	mypb.RegisterMyServiceServer(srv.Gsrv, &myHandler)
//	....
//	srv.Start()
//
// Clients should connect to the server with no security:
//
//	conn, err := grpc.Dial(srv.Addr, grpc.WithInsecure())
//	...
type Server struct {
	Addr string
	Port int
	l    net.Listener
	Gsrv *grpc.Server
}

// NewServer creates a new Server. The Server will be listening for gRPC connections
// at the address named by the Addr field, without TLS.
func NewServer(opts ...grpc.ServerOption) (*Server, error) {
	return NewServerWithPort(0, opts...)
}

// NewServerWithPort creates a new Server at a specific port. The Server will be listening
// for gRPC connections at the address named by the Addr field, without TLS.
func NewServerWithPort(port int, opts ...grpc.ServerOption) (*Server, error) {
	l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port))
	if err != nil {
		return nil, err
	}
	s := &Server{
		Addr: l.Addr().String(),
		Port: parsePort(l.Addr().String()),
		l:    l,
		Gsrv: grpc.NewServer(opts...),
	}
	return s, nil
}

// Start causes the server to start accepting incoming connections.
// Call Start after registering handlers.
func (s *Server) Start() {
	go func() {
		if err := s.Gsrv.Serve(s.l); err != nil {
			log.Printf("testutil.Server.Start: %v", err)
		}
	}()
}

// Close shuts down the server.
func (s *Server) Close() {
	s.Gsrv.Stop()
	s.l.Close()
}

// PageBounds converts an incoming page size and token from an RPC request into
// slice bounds and the outgoing next-page token.
//
// PageBounds assumes that the complete, unpaginated list of items exists as a
// single slice. In addition to the page size and token, PageBounds needs the
// length of that slice.
//
// PageBounds's first two return values should be used to construct a sub-slice of
// the complete, unpaginated slice. E.g. if the complete slice is s, then
// s[from:to] is the desired page. Its third return value should be set as the
// NextPageToken field of the RPC response.
func PageBounds(pageSize int, pageToken string, length int) (from, to int, nextPageToken string, err error) {
	from, to = 0, length
	if pageToken != "" {
		from, err = strconv.Atoi(pageToken)
		if err != nil {
			return 0, 0, "", status.Errorf(codes.InvalidArgument, "bad page token: %v", err)
		}
		if from >= length {
			return length, length, "", nil
		}
	}
	if pageSize > 0 && from+pageSize < length {
		to = from + pageSize
		nextPageToken = strconv.Itoa(to)
	}
	return from, to, nextPageToken, nil
}

var portParser = regexp.MustCompile(`:[0-9]+`)

func parsePort(addr string) int {
	res := portParser.FindAllString(addr, -1)
	if len(res) == 0 {
		panic(fmt.Errorf("parsePort: found no numbers in %s", addr))
	}
	stringPort := res[0][1:] // strip the :
	p, err := strconv.ParseInt(stringPort, 10, 32)
	if err != nil {
		panic(err)
	}
	return int(p)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package version records versioning information about this module.
package version

import (
	"fmt"
	"strings"
)

// These constants determine the current version of this module.
//
// For our release process, we enforce the following rules:
//   - Tagged releases use a tag that is identical to String.
//   - Tagged releases never reference a commit where the String
//     contains "devel".
//   - The set of all commits in this repository where String
//     does not contain "devel" must have a unique String.
//
// Steps for tagging a new release:
//
//  1. Create a new CL.
//
//  2. Update Minor, Patch, and/or PreRelease as necessary.
//     PreRelease must not contain the string "devel".
//
//  3. Since the last released minor version, have there been any changes to
//     generator that relies on new functionality in the runtime?
//     If yes, then increment RequiredGenerated.
//
//  4. Since the last released minor version, have there been any changes to
//     the runtime that removes support for old .pb.go source code?
//     If yes, then increment SupportMinimum.
//
//  5. Send out the CL for review and submit it.
//     Note that the next CL in step 8 must be submitted after this CL
//     without any other CLs in-between.
//
//  6. Tag a new version, where the tag is is the current String.
//
//  7. Write release notes for all notable changes
//     between this release and the last release.
//
//  8. Create a new CL.
//
//  9. Update PreRelease to include the string "devel".
//     For example: "" -> "devel" or "rc.1" -> "rc.1.devel"
//
//  10. Send out the CL for review and submit it.
const (
	Major      = 1
	Minor      = 28
	Patch      = 1
	PreRelease = ""
)

// String formats the version string for this module in semver format.
//
// Examples:
//
//	v1.20.1
//	v1.21.0-rc.1
func String() string {
	v := fmt.Sprintf("v%d.%d.%d", Major, Minor, Patch)
	if PreRelease != "" {
		v += "-" + PreRelease

		// TODO: Add metadata about the commit or build hash.
		// See https://golang.org/issue/29814
		// See https://golang.org/issue/33533
		var metadata string
		if strings.Contains(PreRelease, "devel") && metadata != "" {
			v += "+" + metadata
		}
	}
	return v
}
<commit_msg>all: start v1.28.1-devel<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package version records versioning information about this module.
package version

import (
	"fmt"
	"strings"
)

// These constants determine the current version of this module.
//
// For our release process, we enforce the following rules:
//   - Tagged releases use a tag that is identical to String.
//   - Tagged releases never reference a commit where the String
//     contains "devel".
//   - The set of all commits in this repository where String
//     does not contain "devel" must have a unique String.
//
// Steps for tagging a new release:
//
//  1. Create a new CL.
//
//  2. Update Minor, Patch, and/or PreRelease as necessary.
//     PreRelease must not contain the string "devel".
//
//  3. Since the last released minor version, have there been any changes to
//     generator that relies on new functionality in the runtime?
//     If yes, then increment RequiredGenerated.
//
//  4. Since the last released minor version, have there been any changes to
//     the runtime that removes support for old .pb.go source code?
//     If yes, then increment SupportMinimum.
//
//  5. Send out the CL for review and submit it.
//     Note that the next CL in step 8 must be submitted after this CL
//     without any other CLs in-between.
//
//  6. Tag a new version, where the tag is is the current String.
//
//  7. Write release notes for all notable changes
//     between this release and the last release.
//
//  8. Create a new CL.
//
//  9. Update PreRelease to include the string "devel".
//     For example: "" -> "devel" or "rc.1" -> "rc.1.devel"
//
//  10. Send out the CL for review and submit it.
const (
	Major      = 1
	Minor      = 28
	Patch      = 1
	PreRelease = "devel"
)

// String formats the version string for this module in semver format.
//
// Examples:
//
//	v1.20.1
//	v1.21.0-rc.1
func String() string {
	v := fmt.Sprintf("v%d.%d.%d", Major, Minor, Patch)
	if PreRelease != "" {
		v += "-" + PreRelease

		// TODO: Add metadata about the commit or build hash.
		// See https://golang.org/issue/29814
		// See https://golang.org/issue/33533
		var metadata string
		if strings.Contains(PreRelease, "devel") && metadata != "" {
			v += "+" + metadata
		}
	}
	return v
}
<|endoftext|>


<commit_before>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package profiles

import (
	"fmt"

	"github.com/kris-nova/kubicorn/apis/cluster"
	"github.com/kris-nova/kubicorn/cutil/kubeadm"
)

// NewCentosDigitalOceanCluster creates a basic CentOS DigitalOcean cluster.
func NewCentosDigitalOceanCluster(name string) *cluster.Cluster {
	return &cluster.Cluster{
		Name:     name,
		Cloud:    cluster.CloudDigitalOcean,
		Location: "sfo2",
		SSH: &cluster.SSH{
			PublicKeyPath: "~/.ssh/id_rsa.pub",
			User:          "root",
		},
		KubernetesAPI: &cluster.KubernetesAPI{
			Port: "443",
		},
		Values: &cluster.Values{
			ItemMap: map[string]string{
				"INJECTEDTOKEN": kubeadm.GetRandomToken(),
			},
		},
		ServerPools: []*cluster.ServerPool{
			{
				Type:     cluster.ServerPoolTypeMaster,
				Name:     fmt.Sprintf("%s-master", name),
				MaxCount: 1,
				Image:    "centos-7-x64",
				Size:     "1gb",
				BootstrapScripts: []string{
					"vpn/openvpnMaster-centos.sh",
					"digitalocean_k8s_centos_7_master.sh",
				},
			},
			{
				Type:     cluster.ServerPoolTypeNode,
				Name:     fmt.Sprintf("%s-node", name),
				MaxCount: 1,
				Image:    "centos-7-x64",
				Size:     "1gb",
				BootstrapScripts: []string{
					"vpn/openvpnNode-centos.sh",
					"digitalocean_k8s_centos_7_node.sh",
				},
			},
		},
	}
}
<commit_msg>increase DO centos master memory to 2 GB<commit_after>// Copyright © 2017 The Kubicorn Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package profiles

import (
	"fmt"

	"github.com/kris-nova/kubicorn/apis/cluster"
	"github.com/kris-nova/kubicorn/cutil/kubeadm"
)

// NewCentosDigitalOceanCluster creates a basic CentOS DigitalOcean cluster.
func NewCentosDigitalOceanCluster(name string) *cluster.Cluster {
	return &cluster.Cluster{
		Name:     name,
		Cloud:    cluster.CloudDigitalOcean,
		Location: "sfo2",
		SSH: &cluster.SSH{
			PublicKeyPath: "~/.ssh/id_rsa.pub",
			User:          "root",
		},
		KubernetesAPI: &cluster.KubernetesAPI{
			Port: "443",
		},
		Values: &cluster.Values{
			ItemMap: map[string]string{
				"INJECTEDTOKEN": kubeadm.GetRandomToken(),
			},
		},
		ServerPools: []*cluster.ServerPool{
			{
				Type:     cluster.ServerPoolTypeMaster,
				Name:     fmt.Sprintf("%s-master", name),
				MaxCount: 1,
				Image:    "centos-7-x64",
				Size:     "2gb",
				BootstrapScripts: []string{
					"vpn/openvpnMaster-centos.sh",
					"digitalocean_k8s_centos_7_master.sh",
				},
			},
			{
				Type:     cluster.ServerPoolTypeNode,
				Name:     fmt.Sprintf("%s-node", name),
				MaxCount: 1,
				Image:    "centos-7-x64",
				Size:     "1gb",
				BootstrapScripts: []string{
					"vpn/openvpnNode-centos.sh",
					"digitalocean_k8s_centos_7_node.sh",
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package crawler

import (
	"context"

	"github.com/ipfs-search/ipfs-search/components/index"
	index_types "github.com/ipfs-search/ipfs-search/components/index/types"
	t "github.com/ipfs-search/ipfs-search/types"
)

type existingItem struct {
	*t.AnnotatedResource
	index.Index
	*index_types.Update
}

func (c *Crawler) getExistingItem(ctx context.Context, r *t.AnnotatedResource) (*existingItem, error) {
	indexes := []index.Index{c.indexes.Files, c.indexes.Directories, c.indexes.Invalids, c.indexes.Partials}

	update := new(index_types.Update)

	index, err := index.MultiGet(ctx, indexes, r.ID, update, "references", "last-seen")
	if err != nil {
		return nil, err
	}

	if index == nil {
		// Not found
		return nil, nil
	}

	return &existingItem{
		r, index, update,
	}, nil
}
<commit_msg>Static allocation for updates.<commit_after>package crawler

import (
	"context"

	"github.com/ipfs-search/ipfs-search/components/index"
	index_types "github.com/ipfs-search/ipfs-search/components/index/types"
	t "github.com/ipfs-search/ipfs-search/types"
)

type existingItem struct {
	*t.AnnotatedResource
	index.Index
	*index_types.Update
}

func (c *Crawler) getExistingItem(ctx context.Context, r *t.AnnotatedResource) (*existingItem, error) {
	indexes := []index.Index{c.indexes.Files, c.indexes.Directories, c.indexes.Invalids, c.indexes.Partials}

	update := &index_types.Update{}

	index, err := index.MultiGet(ctx, indexes, r.ID, update, "references", "last-seen")
	if err != nil {
		return nil, err
	}

	if index == nil {
		// Not found
		return nil, nil
	}

	return &existingItem{
		r, index, update,
	}, nil
}
<|endoftext|>


<commit_before>package rtsengine

import (
	"fmt"
	"runtime"
	"time"
)

// MovementMechanic handles the global movement of units
type MovementMechanic struct {
	// The world map that maintains the terrain and units.
	OurWorld *World

	// Command channel
	CommandChannel chan *WirePacket

	// All players in the game
	Players []IPlayer

	// Pathing systems
	Pathing *AStarPathing

	OurGame *Game

	evah bool
}

// NewMovementMechanic factory
func NewMovementMechanic(world *World, cc chan *WirePacket, players []IPlayer, pathing *AStarPathing, ourgame *Game) *MovementMechanic {
	return &MovementMechanic{world, cc, players, pathing, ourgame, true}
}

func (m *MovementMechanic) name() string {
	return "MovementMechanic"
}

func (m *MovementMechanic) stop() {
	m.evah = false
}

func (m *MovementMechanic) start() {

	for m.evah {
		for _, player := range m.Players {
			unitmap := player.PlayerUnits()
			for _, unit := range unitmap.Map {
				movement := unit.movement()

				// Can/Should we move this unit?
				if movement.CanMove() {
					// As long as the MovementDestination exists and is different than the current location (obviously)
					if movement.CurrentLocation != nil && movement.MovementDestination != nil && !movement.CurrentLocation.Eq(*movement.MovementDestination) {
						pathList, err := m.OurGame.FindPath(movement.CurrentLocation, movement.MovementDestination)
						if err != nil {
							fmt.Print(err)
							continue
						}

						// Remove the unit from the world and skip through the A* path
						m.OurWorld.RemoveAt(unit, movement.CurrentLocation)
						for e := pathList.Front(); e != nil; e = e.Next() {
							square := e.Value.(*Square)

							// Skip the current location
							if square.Locus.Eq(*movement.CurrentLocation) {
								continue
							}

							// Ok take the next point and use that and move to that point.
							// That will be our new current destination
							_ = m.OurWorld.Add(unit, &square.Locus)
							movement.CurrentLocation = &square.Locus
							break
						}

						// Update the last movement time (right now).
						unit.sendPacketToChannel(MoveUnit, m.CommandChannel)
						movement.UpdateLastMovement()

						// Free all squares to the pool.
						m.OurGame.FreeList(pathList)
					}
				} // move?
				runtime.Gosched()
			} // Units...
		} // Players...

		time.Sleep(time.Millisecond * 500) // half a second.
	} // eva...

}
<commit_msg>When pathing if you recieve an error nil out the MovementDestination<commit_after>package rtsengine

import (
	"fmt"
	"runtime"
	"time"
)

// MovementMechanic handles the global movement of units
type MovementMechanic struct {
	// The world map that maintains the terrain and units.
	OurWorld *World

	// Command channel
	CommandChannel chan *WirePacket

	// All players in the game
	Players []IPlayer

	// Pathing systems
	Pathing *AStarPathing

	OurGame *Game

	evah bool
}

// NewMovementMechanic factory
func NewMovementMechanic(world *World, cc chan *WirePacket, players []IPlayer, pathing *AStarPathing, ourgame *Game) *MovementMechanic {
	return &MovementMechanic{world, cc, players, pathing, ourgame, true}
}

func (m *MovementMechanic) name() string {
	return "MovementMechanic"
}

func (m *MovementMechanic) stop() {
	m.evah = false
}

func (m *MovementMechanic) start() {

	for m.evah {
		for _, player := range m.Players {
			unitmap := player.PlayerUnits()
			for _, unit := range unitmap.Map {
				movement := unit.movement()

				// Can/Should we move this unit?
				if movement.CanMove() {
					// As long as the MovementDestination exists and is different than the current location (obviously)
					if movement.CurrentLocation != nil && movement.MovementDestination != nil && !movement.CurrentLocation.Eq(*movement.MovementDestination) {
						pathList, err := m.OurGame.FindPath(movement.CurrentLocation, movement.MovementDestination)
						if err != nil {
							movement.MovementDestination = nil
							fmt.Print(err)
							continue
						}

						// Remove the unit from the world and skip through the A* path
						m.OurWorld.RemoveAt(unit, movement.CurrentLocation)
						for e := pathList.Front(); e != nil; e = e.Next() {
							square := e.Value.(*Square)

							// Skip the current location
							if square.Locus.Eq(*movement.CurrentLocation) {
								continue
							}

							// Ok take the next point and use that and move to that point.
							// That will be our new current destination
							_ = m.OurWorld.Add(unit, &square.Locus)
							movement.CurrentLocation = &square.Locus
							break
						}

						// Update the last movement time (right now).
						unit.sendPacketToChannel(MoveUnit, m.CommandChannel)
						movement.UpdateLastMovement()

						// Free all squares to the pool.
						m.OurGame.FreeList(pathList)
					}
				} // move?
				runtime.Gosched()
			} // Units...
		} // Players...

		time.Sleep(time.Millisecond * 500) // half a second.
	} // eva...

}
<|endoftext|>


<commit_before>package anaconda

import (
	"net/url"
	"strconv"
)

func (a TwitterApi) GetBlocksList(v url.Values) (c UserCursor, err error) {
	response_ch := make(chan response)
	a.queryQueue <- query{BaseUrl + "/blocks/list.json", v, &c, _GET, response_ch}
	return c, (<-response_ch).err
}

func (a TwitterApi) GetBlocksIds(v url.Values) (c Cursor, err error) {
	response_ch := make(chan response)
	a.queryQueue <- query{BaseUrl + "/blocks/ids.json", v, &c, _GET, response_ch}
	return c, (<-response_ch).err
}

func (a TwitterApi) BlockUser(screenName string, v url.Values) (user User, err error) {
	v = cleanValues(v)
	v.Set("screen_name", screenName)
	return a.Block(v)
}

func (a TwitterApi) BlockUserId(id int64, v url.Values) (user User, err error) {
	v = cleanValues(v)
	v.Set("user_id", strconv.FormatInt(id, 10))
	return a.Block(v)
}

func (a TwitterApi) Block(v url.Values) (user User, err error) {
	response_ch := make(chan response)
	a.queryQueue <- query{BaseUrl + "/blocks/create.json", v, &user, _POST, response_ch}
	return user, (<-response_ch).err
}

func (a TwitterApi) UnBlockUser(screenName string, v url.Values) (user User, err error) {
	v = cleanValues(v)
	v.Set("screen_name", screenName)
	return a.UnBlock(v)
}

func (a TwitterApi) UnBlockUserId(id int64, v url.Values) (user User, err error) {
	v = cleanValues(v)
	v.Set("user_id", strconv.FormatInt(id, 10))
	return a.UnBlock(v)
}

func (a TwitterApi) UnBlock(v url.Values) (user User, err error) {
	response_ch := make(chan response)
	a.queryQueue <- query{BaseUrl + "/blocks/destroy.json", v, &user, _POST, response_ch}
	return user, (<-response_ch).err
}
<commit_msg>Change spelling from UnBlock to Unblock<commit_after>package anaconda

import (
	"net/url"
	"strconv"
)

func (a TwitterApi) GetBlocksList(v url.Values) (c UserCursor, err error) {
	response_ch := make(chan response)
	a.queryQueue <- query{BaseUrl + "/blocks/list.json", v, &c, _GET, response_ch}
	return c, (<-response_ch).err
}

func (a TwitterApi) GetBlocksIds(v url.Values) (c Cursor, err error) {
	response_ch := make(chan response)
	a.queryQueue <- query{BaseUrl + "/blocks/ids.json", v, &c, _GET, response_ch}
	return c, (<-response_ch).err
}

func (a TwitterApi) BlockUser(screenName string, v url.Values) (user User, err error) {
	v = cleanValues(v)
	v.Set("screen_name", screenName)
	return a.Block(v)
}

func (a TwitterApi) BlockUserId(id int64, v url.Values) (user User, err error) {
	v = cleanValues(v)
	v.Set("user_id", strconv.FormatInt(id, 10))
	return a.Block(v)
}

func (a TwitterApi) Block(v url.Values) (user User, err error) {
	response_ch := make(chan response)
	a.queryQueue <- query{BaseUrl + "/blocks/create.json", v, &user, _POST, response_ch}
	return user, (<-response_ch).err
}

func (a TwitterApi) UnblockUser(screenName string, v url.Values) (user User, err error) {
	v = cleanValues(v)
	v.Set("screen_name", screenName)
	return a.Unblock(v)
}

func (a TwitterApi) UnblockUserId(id int64, v url.Values) (user User, err error) {
	v = cleanValues(v)
	v.Set("user_id", strconv.FormatInt(id, 10))
	return a.Unblock(v)
}

func (a TwitterApi) Unblock(v url.Values) (user User, err error) {
	response_ch := make(chan response)
	a.queryQueue <- query{BaseUrl + "/blocks/destroy.json", v, &user, _POST, response_ch}
	return user, (<-response_ch).err
}
<|endoftext|>


<commit_before>package registry

import (
	"v2ray.com/core/common/loader"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/any"
)

var (
	inboundConfigCreatorCache  = loader.ConfigCreatorCache{}
	outboundConfigCreatorCache = loader.ConfigCreatorCache{}
)

func RegisterInboundConfig(protocol string, creator loader.ConfigCreator) error {
	return inboundConfigCreatorCache.RegisterCreator(protocol, creator)
}

func RegisterOutboundConfig(protocol string, creator loader.ConfigCreator) error {
	return outboundConfigCreatorCache.RegisterCreator(protocol, creator)
}

func MarshalInboundConfig(protocol string, settings *any.Any) (interface{}, error) {
	config, err := inboundConfigCreatorCache.CreateConfig(protocol)
	if err != nil {
		return nil, err
	}
	if err := ptypes.UnmarshalAny(settings, config.(proto.Message)); err != nil {
		return nil, err
	}
	return config, nil
}

func MarshalOutboundConfig(protocol string, settings *any.Any) (interface{}, error) {
	config, err := outboundConfigCreatorCache.CreateConfig(protocol)
	if err != nil {
		return nil, err
	}
	if err := ptypes.UnmarshalAny(settings, config.(proto.Message)); err != nil {
		return nil, err
	}
	return config, nil
}
<commit_msg>check nil settings<commit_after>package registry

import (
	"v2ray.com/core/common/loader"

	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	"github.com/golang/protobuf/ptypes/any"
)

var (
	inboundConfigCreatorCache  = loader.ConfigCreatorCache{}
	outboundConfigCreatorCache = loader.ConfigCreatorCache{}
)

func RegisterInboundConfig(protocol string, creator loader.ConfigCreator) error {
	return inboundConfigCreatorCache.RegisterCreator(protocol, creator)
}

func RegisterOutboundConfig(protocol string, creator loader.ConfigCreator) error {
	return outboundConfigCreatorCache.RegisterCreator(protocol, creator)
}

func MarshalInboundConfig(protocol string, settings *any.Any) (interface{}, error) {
	config, err := inboundConfigCreatorCache.CreateConfig(protocol)
	if err != nil {
		return nil, err
	}
	if settings == nil {
		return config, nil
	}
	if err := ptypes.UnmarshalAny(settings, config.(proto.Message)); err != nil {
		return nil, err
	}
	return config, nil
}

func MarshalOutboundConfig(protocol string, settings *any.Any) (interface{}, error) {
	config, err := outboundConfigCreatorCache.CreateConfig(protocol)
	if err != nil {
		return nil, err
	}
	if settings == nil {
		return config, nil
	}
	if err := ptypes.UnmarshalAny(settings, config.(proto.Message)); err != nil {
		return nil, err
	}
	return config, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"

	"net/http"
	"strconv"

	"github.com/vmware/harbor/src/common/dao"
	"github.com/vmware/harbor/src/common/models"
	"github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/replication/core"
	rep_models "github.com/vmware/harbor/src/replication/models"
	api_models "github.com/vmware/harbor/src/ui/api/models"
	"github.com/vmware/harbor/src/ui/promgr"
)

// RepPolicyAPI handles /api/replicationPolicies /api/replicationPolicies/:id/enablement
type RepPolicyAPI struct {
	BaseController
}

// Prepare validates whether the user has system admin role
func (pa *RepPolicyAPI) Prepare() {
	pa.BaseController.Prepare()
	if !pa.SecurityCtx.IsAuthenticated() {
		pa.HandleUnauthorized()
		return
	}

	if !(pa.Ctx.Request.Method == http.MethodGet || pa.SecurityCtx.IsSysAdmin()) {
		pa.HandleForbidden(pa.SecurityCtx.GetUsername())
		return
	}
}

// Get ...
func (pa *RepPolicyAPI) Get() {
	id := pa.GetIDFromURL()
	policy, err := core.GlobalController.GetPolicy(id)
	if err != nil {
		log.Errorf("failed to get policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	if policy.ID == 0 {
		pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	if !pa.SecurityCtx.HasAllPerm(policy.ProjectIDs[0]) {
		pa.HandleForbidden(pa.SecurityCtx.GetUsername())
		return
	}

	ply, err := convertFromRepPolicy(pa.ProjectMgr, policy)
	if err != nil {
		pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
		return
	}

	pa.Data["json"] = ply
	pa.ServeJSON()
}

// List ...
func (pa *RepPolicyAPI) List() {
	queryParam := rep_models.QueryParameter{
		Name: pa.GetString("name"),
	}
	projectIDStr := pa.GetString("project_id")
	if len(projectIDStr) > 0 {
		projectID, err := strconv.ParseInt(projectIDStr, 10, 64)
		if err != nil || projectID <= 0 {
			pa.CustomAbort(http.StatusBadRequest, "invalid project ID")
		}
		queryParam.ProjectID = projectID
	}
	queryParam.Page, queryParam.PageSize = pa.GetPaginationParams()

	result, err := core.GlobalController.GetPolicies(queryParam)
	if err != nil {
		log.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam)
		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	var total int64
	policies := []*api_models.ReplicationPolicy{}
	if result != nil {
		total = result.Total
		for _, policy := range result.Policies {
			if !pa.SecurityCtx.HasAllPerm(policy.ProjectIDs[0]) {
				continue
			}
			ply, err := convertFromRepPolicy(pa.ProjectMgr, *policy)
			if err != nil {
				pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
				return
			}
			policies = append(policies, ply)
		}
	}

	pa.SetPaginationHeader(total, queryParam.Page, queryParam.PageSize)

	pa.Data["json"] = policies
	pa.ServeJSON()
}

// Post creates a replicartion policy
func (pa *RepPolicyAPI) Post() {
	policy := &api_models.ReplicationPolicy{}
	pa.DecodeJSONReqAndValidate(policy)

	// check the existence of projects
	for _, project := range policy.Projects {
		pro, err := pa.ProjectMgr.Get(project.ProjectID)
		if err != nil {
			pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
			return
		}
		if pro == nil {
			pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID))
			return
		}
		project.Name = pro.Name
	}

	// check the existence of targets
	for _, target := range policy.Targets {
		t, err := dao.GetRepTarget(target.ID)
		if err != nil {
			pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err))
			return
		}

		if t == nil {
			pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID))
			return
		}
	}

	id, err := core.GlobalController.CreatePolicy(convertToRepPolicy(policy))
	if err != nil {
		pa.HandleInternalServerError(fmt.Sprintf("failed to create policy: %v", err))
		return
	}

	if policy.ReplicateExistingImageNow {
		go func() {
			if err = startReplication(id); err != nil {
				log.Errorf("failed to send replication signal for policy %d: %v", id, err)
				return
			}
			log.Infof("replication signal for policy %d sent", id)
		}()
	}

	pa.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}

// Put updates the replication policy
func (pa *RepPolicyAPI) Put() {
	id := pa.GetIDFromURL()

	originalPolicy, err := core.GlobalController.GetPolicy(id)
	if err != nil {
		log.Errorf("failed to get policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	if originalPolicy.ID == 0 {
		pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	policy := &api_models.ReplicationPolicy{}
	pa.DecodeJSONReqAndValidate(policy)

	policy.ID = id

	// check the existence of projects
	for _, project := range policy.Projects {
		pro, err := pa.ProjectMgr.Get(project.ProjectID)
		if err != nil {
			pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
			return
		}
		if pro == nil {
			pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID))
			return
		}
		project.Name = pro.Name
	}

	// check the existence of targets
	for _, target := range policy.Targets {
		t, err := dao.GetRepTarget(target.ID)
		if err != nil {
			pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err))
			return
		}

		if t == nil {
			pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID))
			return
		}
	}

	if err = core.GlobalController.UpdatePolicy(convertToRepPolicy(policy)); err != nil {
		pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err))
		return
	}

	if policy.ReplicateExistingImageNow {
		go func() {
			if err = startReplication(id); err != nil {
				log.Errorf("failed to send replication signal for policy %d: %v", id, err)
				return
			}
			log.Infof("replication signal for policy %d sent", id)
		}()
	}
}

// Delete the replication policy
func (pa *RepPolicyAPI) Delete() {
	id := pa.GetIDFromURL()

	policy, err := core.GlobalController.GetPolicy(id)
	if err != nil {
		log.Errorf("failed to get policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	if policy.ID == 0 {
		pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	count, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
		PolicyID: id,
		Statuses: []string{models.JobRunning, models.JobRetrying, models.JobPending},
	})
	if err != nil {
		log.Errorf("failed to filter jobs of policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, "")
	}
	if count > 0 {
		pa.CustomAbort(http.StatusPreconditionFailed, "policy has running/retrying/pending jobs, can not be deleted")
	}

	if err = core.GlobalController.RemovePolicy(id); err != nil {
		log.Errorf("failed to delete policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, "")
	}
}

func convertFromRepPolicy(projectMgr promgr.ProjectManager, policy rep_models.ReplicationPolicy) (*api_models.ReplicationPolicy, error) {
	if policy.ID == 0 {
		return nil, nil
	}

	// populate simple properties
	ply := &api_models.ReplicationPolicy{
		ID:                policy.ID,
		Name:              policy.Name,
		Description:       policy.Description,
		Filters:           policy.Filters,
		ReplicateDeletion: policy.ReplicateDeletion,
		Trigger:           policy.Trigger,
		CreationTime:      policy.CreationTime,
		UpdateTime:        policy.UpdateTime,
	}

	// populate projects
	for _, projectID := range policy.ProjectIDs {
		project, err := projectMgr.Get(projectID)
		if err != nil {
			return nil, err
		}

		ply.Projects = append(ply.Projects, project)
	}

	// populate targets
	for _, targetID := range policy.TargetIDs {
		target, err := dao.GetRepTarget(targetID)
		if err != nil {
			return nil, err
		}
		target.Password = ""
		ply.Targets = append(ply.Targets, target)
	}

	// TODO call the method from replication controller
	errJobCount, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
		PolicyID: policy.ID,
		Statuses: []string{models.JobError},
	})
	if err != nil {
		return nil, err
	}
	ply.ErrorJobCount = errJobCount

	return ply, nil
}

func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.ReplicationPolicy {
	if policy == nil {
		return rep_models.ReplicationPolicy{}
	}

	ply := rep_models.ReplicationPolicy{
		ID:                policy.ID,
		Name:              policy.Name,
		Description:       policy.Description,
		Filters:           policy.Filters,
		ReplicateDeletion: policy.ReplicateDeletion,
		Trigger:           policy.Trigger,
		CreationTime:      policy.CreationTime,
		UpdateTime:        policy.UpdateTime,
	}

	for _, project := range policy.Projects {
		ply.ProjectIDs = append(ply.ProjectIDs, project.ProjectID)
		ply.Namespaces = append(ply.Namespaces, project.Name)
	}

	for _, target := range policy.Targets {
		ply.TargetIDs = append(ply.TargetIDs, target.ID)
	}

	return ply
}
<commit_msg>Fix bug #4819: the scheduled rule can not be removed<commit_after>// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package api

import (
	"fmt"

	"net/http"
	"strconv"

	"github.com/vmware/harbor/src/common/dao"
	"github.com/vmware/harbor/src/common/models"
	"github.com/vmware/harbor/src/common/utils/log"
	"github.com/vmware/harbor/src/replication/core"
	rep_models "github.com/vmware/harbor/src/replication/models"
	api_models "github.com/vmware/harbor/src/ui/api/models"
	"github.com/vmware/harbor/src/ui/promgr"
)

// RepPolicyAPI handles /api/replicationPolicies /api/replicationPolicies/:id/enablement
type RepPolicyAPI struct {
	BaseController
}

// Prepare validates whether the user has system admin role
func (pa *RepPolicyAPI) Prepare() {
	pa.BaseController.Prepare()
	if !pa.SecurityCtx.IsAuthenticated() {
		pa.HandleUnauthorized()
		return
	}

	if !(pa.Ctx.Request.Method == http.MethodGet || pa.SecurityCtx.IsSysAdmin()) {
		pa.HandleForbidden(pa.SecurityCtx.GetUsername())
		return
	}
}

// Get ...
func (pa *RepPolicyAPI) Get() {
	id := pa.GetIDFromURL()
	policy, err := core.GlobalController.GetPolicy(id)
	if err != nil {
		log.Errorf("failed to get policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	if policy.ID == 0 {
		pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	if !pa.SecurityCtx.HasAllPerm(policy.ProjectIDs[0]) {
		pa.HandleForbidden(pa.SecurityCtx.GetUsername())
		return
	}

	ply, err := convertFromRepPolicy(pa.ProjectMgr, policy)
	if err != nil {
		pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
		return
	}

	pa.Data["json"] = ply
	pa.ServeJSON()
}

// List ...
func (pa *RepPolicyAPI) List() {
	queryParam := rep_models.QueryParameter{
		Name: pa.GetString("name"),
	}
	projectIDStr := pa.GetString("project_id")
	if len(projectIDStr) > 0 {
		projectID, err := strconv.ParseInt(projectIDStr, 10, 64)
		if err != nil || projectID <= 0 {
			pa.CustomAbort(http.StatusBadRequest, "invalid project ID")
		}
		queryParam.ProjectID = projectID
	}
	queryParam.Page, queryParam.PageSize = pa.GetPaginationParams()

	result, err := core.GlobalController.GetPolicies(queryParam)
	if err != nil {
		log.Errorf("failed to get policies: %v, query parameters: %v", err, queryParam)
		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	var total int64
	policies := []*api_models.ReplicationPolicy{}
	if result != nil {
		total = result.Total
		for _, policy := range result.Policies {
			if !pa.SecurityCtx.HasAllPerm(policy.ProjectIDs[0]) {
				continue
			}
			ply, err := convertFromRepPolicy(pa.ProjectMgr, *policy)
			if err != nil {
				pa.ParseAndHandleError(fmt.Sprintf("failed to convert from replication policy"), err)
				return
			}
			policies = append(policies, ply)
		}
	}

	pa.SetPaginationHeader(total, queryParam.Page, queryParam.PageSize)

	pa.Data["json"] = policies
	pa.ServeJSON()
}

// Post creates a replicartion policy
func (pa *RepPolicyAPI) Post() {
	policy := &api_models.ReplicationPolicy{}
	pa.DecodeJSONReqAndValidate(policy)

	// check the existence of projects
	for _, project := range policy.Projects {
		pro, err := pa.ProjectMgr.Get(project.ProjectID)
		if err != nil {
			pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
			return
		}
		if pro == nil {
			pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID))
			return
		}
		project.Name = pro.Name
	}

	// check the existence of targets
	for _, target := range policy.Targets {
		t, err := dao.GetRepTarget(target.ID)
		if err != nil {
			pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err))
			return
		}

		if t == nil {
			pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID))
			return
		}
	}

	id, err := core.GlobalController.CreatePolicy(convertToRepPolicy(policy))
	if err != nil {
		pa.HandleInternalServerError(fmt.Sprintf("failed to create policy: %v", err))
		return
	}

	if policy.ReplicateExistingImageNow {
		go func() {
			if err = startReplication(id); err != nil {
				log.Errorf("failed to send replication signal for policy %d: %v", id, err)
				return
			}
			log.Infof("replication signal for policy %d sent", id)
		}()
	}

	pa.Redirect(http.StatusCreated, strconv.FormatInt(id, 10))
}

// Put updates the replication policy
func (pa *RepPolicyAPI) Put() {
	id := pa.GetIDFromURL()

	originalPolicy, err := core.GlobalController.GetPolicy(id)
	if err != nil {
		log.Errorf("failed to get policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	if originalPolicy.ID == 0 {
		pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	policy := &api_models.ReplicationPolicy{}
	pa.DecodeJSONReqAndValidate(policy)

	policy.ID = id

	// check the existence of projects
	for _, project := range policy.Projects {
		pro, err := pa.ProjectMgr.Get(project.ProjectID)
		if err != nil {
			pa.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %d", project.ProjectID), err)
			return
		}
		if pro == nil {
			pa.HandleNotFound(fmt.Sprintf("project %d not found", project.ProjectID))
			return
		}
		project.Name = pro.Name
	}

	// check the existence of targets
	for _, target := range policy.Targets {
		t, err := dao.GetRepTarget(target.ID)
		if err != nil {
			pa.HandleInternalServerError(fmt.Sprintf("failed to get target %d: %v", target.ID, err))
			return
		}

		if t == nil {
			pa.HandleNotFound(fmt.Sprintf("target %d not found", target.ID))
			return
		}
	}

	if err = core.GlobalController.UpdatePolicy(convertToRepPolicy(policy)); err != nil {
		pa.HandleInternalServerError(fmt.Sprintf("failed to update policy %d: %v", id, err))
		return
	}

	if policy.ReplicateExistingImageNow {
		go func() {
			if err = startReplication(id); err != nil {
				log.Errorf("failed to send replication signal for policy %d: %v", id, err)
				return
			}
			log.Infof("replication signal for policy %d sent", id)
		}()
	}
}

// Delete the replication policy
func (pa *RepPolicyAPI) Delete() {
	id := pa.GetIDFromURL()

	policy, err := core.GlobalController.GetPolicy(id)
	if err != nil {
		log.Errorf("failed to get policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	if policy.ID == 0 {
		pa.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	count, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
		PolicyID: id,
		Statuses: []string{models.JobRunning, models.JobRetrying, models.JobPending},
		// only get the transfer and delete jobs, do not get schedule job
		Operations: []string{models.RepOpTransfer, models.RepOpDelete},
	})
	if err != nil {
		log.Errorf("failed to filter jobs of policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, "")
	}
	if count > 0 {
		pa.CustomAbort(http.StatusPreconditionFailed, "policy has running/retrying/pending jobs, can not be deleted")
	}

	if err = core.GlobalController.RemovePolicy(id); err != nil {
		log.Errorf("failed to delete policy %d: %v", id, err)
		pa.CustomAbort(http.StatusInternalServerError, "")
	}
}

func convertFromRepPolicy(projectMgr promgr.ProjectManager, policy rep_models.ReplicationPolicy) (*api_models.ReplicationPolicy, error) {
	if policy.ID == 0 {
		return nil, nil
	}

	// populate simple properties
	ply := &api_models.ReplicationPolicy{
		ID:                policy.ID,
		Name:              policy.Name,
		Description:       policy.Description,
		Filters:           policy.Filters,
		ReplicateDeletion: policy.ReplicateDeletion,
		Trigger:           policy.Trigger,
		CreationTime:      policy.CreationTime,
		UpdateTime:        policy.UpdateTime,
	}

	// populate projects
	for _, projectID := range policy.ProjectIDs {
		project, err := projectMgr.Get(projectID)
		if err != nil {
			return nil, err
		}

		ply.Projects = append(ply.Projects, project)
	}

	// populate targets
	for _, targetID := range policy.TargetIDs {
		target, err := dao.GetRepTarget(targetID)
		if err != nil {
			return nil, err
		}
		target.Password = ""
		ply.Targets = append(ply.Targets, target)
	}

	// TODO call the method from replication controller
	errJobCount, err := dao.GetTotalCountOfRepJobs(&models.RepJobQuery{
		PolicyID: policy.ID,
		Statuses: []string{models.JobError},
	})
	if err != nil {
		return nil, err
	}
	ply.ErrorJobCount = errJobCount

	return ply, nil
}

func convertToRepPolicy(policy *api_models.ReplicationPolicy) rep_models.ReplicationPolicy {
	if policy == nil {
		return rep_models.ReplicationPolicy{}
	}

	ply := rep_models.ReplicationPolicy{
		ID:                policy.ID,
		Name:              policy.Name,
		Description:       policy.Description,
		Filters:           policy.Filters,
		ReplicateDeletion: policy.ReplicateDeletion,
		Trigger:           policy.Trigger,
		CreationTime:      policy.CreationTime,
		UpdateTime:        policy.UpdateTime,
	}

	for _, project := range policy.Projects {
		ply.ProjectIDs = append(ply.ProjectIDs, project.ProjectID)
		ply.Namespaces = append(ply.Namespaces, project.Name)
	}

	for _, target := range policy.Targets {
		ply.TargetIDs = append(ply.TargetIDs, target.ID)
	}

	return ply
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"time"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/gogo/protobuf/proto"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/worker"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps/server"
	"google.golang.org/grpc"
)

// appEnv stores the environment variables that this worker needs
type appEnv struct {
	// Address of etcd, so that worker can write its own IP there for discoverh
	EtcdAddress string `env:"ETCD_PORT_2379_TCP_ADDR,required"`

	// Address for connecting to pachd (so this can download input data)
	PachdAddress string `env:"PACHD_PORT_650_TCP_ADDR,required"`

	// Prefix in etcd for all pachd-related records
	PPSPrefix string `env:"PPS_ETCD_PREFIX,required"`

	// worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`

	// Either pipeline name or job name must be set
	PPSPipelineName string `env:"PPS_PIPELINE_NAME"`
	PPSJobID        string `env:"PPS_JOB_ID"`
	PodName         string `env:"PPS_POD_NAME,required"`
}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func validateEnv(appEnv *appEnv) error {
	if appEnv.PPSPipelineName == "" && appEnv.PPSJobID == "" {
		return fmt.Errorf("worker must recieve either pipeline name or job ID, but got neither")
	} else if appEnv.PPSPipelineName != "" && appEnv.PPSJobID != "" {
		return fmt.Errorf("worker must recieve either pipeline name or job ID, but got both")
	}
	return nil
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of
func getPipelineInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.PipelineInfo, error) {
	ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "pipelines", appEnv.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline, got %d: %v", len(resp.Kvs), resp)
	}
	pipelineInfo := new(pps.PipelineInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), pipelineInfo); err != nil {
		return nil, err
	}
	return pipelineInfo, nil
}

func getJobInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.JobInfo, error) {
	ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "jobs", appEnv.PPSJobID))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 job, got %d: %v", len(resp.Kvs), resp)
	}
	jobInfo := new(pps.JobInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), jobInfo); err != nil {
		return nil, err
	}
	return jobInfo, nil
}

func do(appEnvObj interface{}) error {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	appEnv := appEnvObj.(*appEnv)
	if err := validateEnv(appEnv); err != nil {
		return err
	}

	// get pachd client, so we can upload output data from the user binary
	pachClient, err := client.NewFromAddress(fmt.Sprintf("%v:650", appEnv.PachdAddress))
	if err != nil {
		return err
	}
	go pachClient.KeepConnected(make(chan bool)) // we never cancel the connection

	// Get etcd client, so we can register our IP (so pachd can discover us)
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{fmt.Sprintf("%s:2379", appEnv.EtcdAddress)},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return err
	}

	// Construct worker API server. Get relevant pipeline or job info, and then
	// use that to create a worker.APIServer.
	var workerRcName string
	var apiServer *worker.APIServer
	if appEnv.PPSPipelineName != "" {
		pipelineInfo, err := getPipelineInfo(etcdClient, appEnv)
		if err != nil {
			return err
		}
		workerRcName = ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
		apiServer = worker.NewPipelineAPIServer(pachClient, pipelineInfo, appEnv.PodName)
	} else if appEnv.PPSJobID != "" {
		jobInfo, err := getJobInfo(etcdClient, appEnv)
		if err != nil {
			return err
		}
		workerRcName = ppsserver.JobRcName(jobInfo.Job.ID)
		apiServer = worker.NewJobAPIServer(pachClient, jobInfo, appEnv.PodName)
	}

	// Setup the hostPath mount to use a unique directory for this worker
	workerDir := filepath.Join(client.PPSHostPath, appEnv.PodName)
	if err := os.MkdirAll(workerDir, 0777); err != nil {
		return err
	}
	if err := os.Symlink(workerDir, client.PPSInputPrefix); err != nil {
		return err
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			func(s *grpc.Server) {
				worker.RegisterWorkerServer(s, apiServer)
				close(ready)
			},
			grpcutil.ServeOptions{
				Version:    version.Version,
				MaxMsgSize: grpcutil.MaxMsgSize,
			},
			grpcutil.ServeEnv{
				GRPCPort: client.PPSWorkerPort,
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(appEnv.PPSPrefix, "workers", workerRcName, appEnv.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
	resp, err := etcdClient.Grant(ctx, 10 /* seconds */)
	if err != nil {
		return err
	}

	// keepalive forever
	if _, err := etcdClient.KeepAlive(context.Background(), resp.ID); err != nil {
		return err
	}

	// Actually write "key" into etcd
	ctx, _ = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	if _, err := etcdClient.Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return err
	}

	// If server ever exits, return error
	if err := eg.Wait(); err != nil {
		return err
	}
	return nil
}
<commit_msg>Use sidecar in worker<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"path"
	"path/filepath"
	"time"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/gogo/protobuf/proto"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/cmdutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/worker"
	ppsserver "github.com/pachyderm/pachyderm/src/server/pps/server"
	"google.golang.org/grpc"
)

// appEnv stores the environment variables that this worker needs
type appEnv struct {
	// Address of etcd, so that worker can write its own IP there for discoverh
	EtcdAddress string `env:"ETCD_PORT_2379_TCP_ADDR,required"`

	// Address for connecting to pachd (so this can download input data)
	PachdAddress string `env:"PACHD_PORT_650_TCP_ADDR,required"`

	// Prefix in etcd for all pachd-related records
	PPSPrefix string `env:"PPS_ETCD_PREFIX,required"`

	// worker gets its own IP here, via the k8s downward API. It then writes that
	// IP back to etcd so that pachd can discover it
	PPSWorkerIP string `env:"PPS_WORKER_IP,required"`

	// Either pipeline name or job name must be set
	PPSPipelineName string `env:"PPS_PIPELINE_NAME"`
	PPSJobID        string `env:"PPS_JOB_ID"`
	PodName         string `env:"PPS_POD_NAME,required"`
}

func main() {
	cmdutil.Main(do, &appEnv{})
}

func validateEnv(appEnv *appEnv) error {
	if appEnv.PPSPipelineName == "" && appEnv.PPSJobID == "" {
		return fmt.Errorf("worker must recieve either pipeline name or job ID, but got neither")
	} else if appEnv.PPSPipelineName != "" && appEnv.PPSJobID != "" {
		return fmt.Errorf("worker must recieve either pipeline name or job ID, but got both")
	}
	return nil
}

// getPipelineInfo gets the PipelineInfo proto describing the pipeline that this
// worker is part of
func getPipelineInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.PipelineInfo, error) {
	ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "pipelines", appEnv.PPSPipelineName))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 pipeline, got %d: %v", len(resp.Kvs), resp)
	}
	pipelineInfo := new(pps.PipelineInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), pipelineInfo); err != nil {
		return nil, err
	}
	return pipelineInfo, nil
}

func getJobInfo(etcdClient *etcd.Client, appEnv *appEnv) (*pps.JobInfo, error) {
	ctx, _ := context.WithTimeout(context.Background(), 30*time.Second)
	resp, err := etcdClient.Get(ctx, path.Join(appEnv.PPSPrefix, "jobs", appEnv.PPSJobID))
	if err != nil {
		return nil, err
	}
	if len(resp.Kvs) != 1 {
		return nil, fmt.Errorf("expected to find 1 job, got %d: %v", len(resp.Kvs), resp)
	}
	jobInfo := new(pps.JobInfo)
	if err := proto.UnmarshalText(string(resp.Kvs[0].Value), jobInfo); err != nil {
		return nil, err
	}
	return jobInfo, nil
}

func do(appEnvObj interface{}) error {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	appEnv := appEnvObj.(*appEnv)
	if err := validateEnv(appEnv); err != nil {
		return err
	}

	// get pachd client, so we can upload output data from the user binary
	pachClient, err := client.NewFromAddress("localhost:650")
	if err != nil {
		return err
	}
	go pachClient.KeepConnected(make(chan bool)) // we never cancel the connection

	// Get etcd client, so we can register our IP (so pachd can discover us)
	etcdClient, err := etcd.New(etcd.Config{
		Endpoints:   []string{fmt.Sprintf("%s:2379", appEnv.EtcdAddress)},
		DialOptions: client.EtcdDialOptions(),
	})
	if err != nil {
		return err
	}

	// Construct worker API server. Get relevant pipeline or job info, and then
	// use that to create a worker.APIServer.
	var workerRcName string
	var apiServer *worker.APIServer
	if appEnv.PPSPipelineName != "" {
		pipelineInfo, err := getPipelineInfo(etcdClient, appEnv)
		if err != nil {
			return err
		}
		workerRcName = ppsserver.PipelineRcName(pipelineInfo.Pipeline.Name, pipelineInfo.Version)
		apiServer = worker.NewPipelineAPIServer(pachClient, pipelineInfo, appEnv.PodName)
	} else if appEnv.PPSJobID != "" {
		jobInfo, err := getJobInfo(etcdClient, appEnv)
		if err != nil {
			return err
		}
		workerRcName = ppsserver.JobRcName(jobInfo.Job.ID)
		apiServer = worker.NewJobAPIServer(pachClient, jobInfo, appEnv.PodName)
	}

	// Setup the hostPath mount to use a unique directory for this worker
	workerDir := filepath.Join(client.PPSHostPath, appEnv.PodName)
	if err := os.MkdirAll(workerDir, 0777); err != nil {
		return err
	}
	if err := os.Symlink(workerDir, client.PPSInputPrefix); err != nil {
		return err
	}

	// Start worker api server
	eg := errgroup.Group{}
	ready := make(chan error)
	eg.Go(func() error {
		return grpcutil.Serve(
			func(s *grpc.Server) {
				worker.RegisterWorkerServer(s, apiServer)
				close(ready)
			},
			grpcutil.ServeOptions{
				Version:    version.Version,
				MaxMsgSize: grpcutil.MaxMsgSize,
			},
			grpcutil.ServeEnv{
				GRPCPort: client.PPSWorkerPort,
			},
		)
	})

	// Wait until server is ready, then put our IP address into etcd, so pachd can
	// discover us
	<-ready
	key := path.Join(appEnv.PPSPrefix, "workers", workerRcName, appEnv.PPSWorkerIP)

	// Prepare to write "key" into etcd by creating lease -- if worker dies, our
	// IP will be removed from etcd
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
	resp, err := etcdClient.Grant(ctx, 10 /* seconds */)
	if err != nil {
		return err
	}

	// keepalive forever
	if _, err := etcdClient.KeepAlive(context.Background(), resp.ID); err != nil {
		return err
	}

	// Actually write "key" into etcd
	ctx, _ = context.WithTimeout(context.Background(), 10*time.Second) // new ctx
	if _, err := etcdClient.Put(ctx, key, "", etcd.WithLease(resp.ID)); err != nil {
		return err
	}

	// If server ever exits, return error
	if err := eg.Wait(); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"

	"github.com/Zac-Garby/pluto/token"
)

type Error struct {
	Message    string
	Start, End token.Position
}

func (p *Parser) err(msg string, start, end token.Position) {
	err := Error{
		Message: msg,
		Start:   start,
		End:     end,
	}

	p.Errors = append(p.Errors, err)
}

func (p *Parser) defaultErr(msg string) {
	err := Error{
		Message: msg,
		Start:   p.cur.Start,
		End:     p.cur.End,
	}

	p.Errors = append(p.Errors, err)
}

func (p *Parser) peekErr(ts ...token.Type) {
	if len(ts) > 1 {
		msg := "expected either "

		for i, t := range ts {
			msg += string(t)

			if i+1 < len(ts) {
				msg += ", "
			} else if i < len(ts) {
				msg += ", or "
			}
		}

		msg += ", but got " + string(p.peek.Type)

		p.err(msg, p.peek.Start, p.peek.End)
	} else if len(ts) == 1 {
		msg := fmt.Sprintf("expected %s, but got %s", ts[0], p.peek.Type)
		p.err(msg, p.peek.Start, p.peek.End)
	}
}

func (p *Parser) curErr(ts ...token.Type) {
	if len(ts) > 1 {
		msg := "expected either "

		for i, t := range ts {
			msg += string(t)

			if i+1 < len(ts) {
				msg += ", "
			} else if i < len(ts) {
				msg += ", or "
			}
		}

		msg += ", but got " + string(p.cur.Type)

		p.err(msg, p.cur.Start, p.cur.End)
	} else if len(ts) == 1 {
		msg := fmt.Sprintf("expected %s, but got %s", ts[0], p.cur.Type)
		p.err(msg, p.cur.Start, p.cur.End)
	}
}

func (p *Parser) unexpectedTokenErr(t token.Type) {
	msg := fmt.Sprintf("unexpected token: %s", t)
	p.defaultErr(msg)
}

func (p *Parser) printError(index int) {
	err := p.Errors[index]

	fmt.Printf("%s to %s -- %s", err.Start.String(), err.End.String(), err.Message)
}

func (p *Parser) printErrors() {
	for i := range p.Errors {
		p.printError(i)
	}
}
<commit_msg>Export Parser.PrintErrors()<commit_after>package parser

import (
	"fmt"

	"github.com/Zac-Garby/pluto/token"
)

type Error struct {
	Message    string
	Start, End token.Position
}

func (p *Parser) err(msg string, start, end token.Position) {
	err := Error{
		Message: msg,
		Start:   start,
		End:     end,
	}

	p.Errors = append(p.Errors, err)
}

func (p *Parser) defaultErr(msg string) {
	err := Error{
		Message: msg,
		Start:   p.cur.Start,
		End:     p.cur.End,
	}

	p.Errors = append(p.Errors, err)
}

func (p *Parser) peekErr(ts ...token.Type) {
	if len(ts) > 1 {
		msg := "expected either "

		for i, t := range ts {
			msg += string(t)

			if i+1 < len(ts) {
				msg += ", "
			} else if i < len(ts) {
				msg += ", or "
			}
		}

		msg += ", but got " + string(p.peek.Type)

		p.err(msg, p.peek.Start, p.peek.End)
	} else if len(ts) == 1 {
		msg := fmt.Sprintf("expected %s, but got %s", ts[0], p.peek.Type)
		p.err(msg, p.peek.Start, p.peek.End)
	}
}

func (p *Parser) curErr(ts ...token.Type) {
	if len(ts) > 1 {
		msg := "expected either "

		for i, t := range ts {
			msg += string(t)

			if i+1 < len(ts) {
				msg += ", "
			} else if i < len(ts) {
				msg += ", or "
			}
		}

		msg += ", but got " + string(p.cur.Type)

		p.err(msg, p.cur.Start, p.cur.End)
	} else if len(ts) == 1 {
		msg := fmt.Sprintf("expected %s, but got %s", ts[0], p.cur.Type)
		p.err(msg, p.cur.Start, p.cur.End)
	}
}

func (p *Parser) unexpectedTokenErr(t token.Type) {
	msg := fmt.Sprintf("unexpected token: %s", t)
	p.defaultErr(msg)
}

func (p *Parser) printError(index int) {
	err := p.Errors[index]

	fmt.Printf("%s to %s -- %s", err.Start.String(), err.End.String(), err.Message)
}

func (p *Parser) PrintErrors() {
	for i := range p.Errors {
		p.printError(i)
	}
}
<|endoftext|>


<commit_before>package parser

import (
	"io"
	"net"
	"net/mail"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"text/scanner"
	"unicode"

	"github.com/yuuki/gokc/log"
)

const (
	EOF = 0
)

var SYMBOL_TABLES = map[string]int{
	"{": LB,
	"}": RB,

	"global_defs":                     GLOBALDEFS,
	"notification_email":              NOTIFICATION_EMAIL,
	"notification_email_from":         NOTIFICATION_EMAIL_FROM,
	"smtp_server":                     SMTP_SERVER,
	"smtp_connect_timeout":            SMTP_CONNECT_TIMEOUT,
	"router_id":                       ROUTER_ID,
	"lvs_id":                          LVS_ID,
	"vrrp_mcast_group4":               VRRP_MCAST_GROUP4,
	"vrrp_mcast_group6":               VRRP_MCAST_GROUP6,
	"vrrp_garp_master_delay":          VRRP_GARP_MASTER_DELAY,
	"vrrp_garp_master_repeat":         VRRP_GARP_MASTER_REPEAT,
	"vrrp_garp_master_refresh":        VRRP_GARP_MASTER_REFRESH,
	"vrrp_garp_master_refresh_repeat": VRRP_GARP_MASTER_REFRESH_REPEAT,
	"vrrp_version":                    VRRP_VERSION,

	"static_ipaddress": STATIC_IPADDRESS,
	"static_routes":    STATIC_ROUTES,
	"static_rules":     STATIC_RULES,

	"vrrp_sync_group": VRRP_SYNC_GROUP,
	"group":           GROUP,

	"vrrp_instance":              VRRP_INSTANCE,
	"use_vmac":                   USE_VMAC,
	"version":                    VERSION,
	"vmac_xmit_base":             VMAC_XMIT_BASE,
	"native_ipv6":                NATIVE_IPV6,
	"interface":                  INTERFACE,
	"mcast_src_ip":               MCAST_SRC_IP,
	"unicast_src_ip":             UNICAST_SRC_IP,
	"unicast_peer":               UNICAST_PEER,
	"lvs_sync_daemon_interface":  LVS_SYNC_DAEMON_INTERFACE,
	"virtual_router_id":          VIRTUAL_ROUTER_ID,
	"nopreempt":                  NOPREEMPT,
	"priority":                   PRIORITY,
	"advert_int":                 ADVERT_INT,
	"virtual_ipaddress":          VIRTUAL_IPADDRESS,
	"virtual_ipaddress_excluded": VIRTUAL_IPADDRESS_EXCLUDED,
	"virtual_routes":             VIRTUAL_ROUTES,
	"state":                      STATE,
	"MASTER":                     MASTER,
	"BACKUP":                     BACKUP,
	"garp_master_delay":          GARP_MASTER_DELAY,
	"smtp_alert":                 SMTP_ALERT,
	"authentication":             AUTHENTICATION,
	"auth_type":                  AUTH_TYPE,
	"auth_pass":                  AUTH_PASS,
	"PASS":                       PASS,
	"AH":                         AH,
	"label":                      LABEL,
	"dev":                        DEV,
	"scope":                      SCOPE,
	"site":                       SITE,
	"link":                       LINK,
	"host":                       HOST,
	"nowhere":                    NOWHERE,
	"global":                     GLOBAL,
	"brd":                        BRD,
	"src":                        SRC,
	"from":                       FROM,
	"to":                         TO,
	"via":                        VIA,
	"gw":                         GW,
	"or":                         OR,
	"table":                      TABLE,
	"metric":                     METRIC,
	"blackhole":                  BLACKHOLE,
	"track_interface":            TRACK_INTERFACE,
	"track_script":               TRACK_SCRIPT,
	"dont_track_primary":         DONT_TRACK_PRIMARY,
	"notify_master":              NOTIFY_MASTER,
	"notify_backup":              NOTIFY_BACKUP,
	"notify_fault":               NOTIFY_FAULT,
	"notify_stop":                NOTIFY_STOP,
	"notify":                     NOTIFY,

	"vrrp_script": VRRP_SCRIPT,
	"script":      SCRIPT,
	"interval":    INTERVAL,
	"timeout":     TIMEOUT,
	"fall":        FALL,
	"rise":        RISE,

	"virtual_server_group": VIRTUAL_SERVER_GROUP,
	"fwmark":               FWMARK,

	"virtual_server": VIRTUAL_SERVER,
	"delay_loop":     DELAY_LOOP,
	"lb_algo":        LB_ALGO,
	"lb_kind":        LB_KIND,
	"lvs_sched":      LVS_SCHED,
	"lvs_method":     LVS_METHOD,
	"rr":             RR,
	"wrr":            WRR,
	"lc":             LC,
	"wlc":            WLC,
	"fo":             FO,
	"ovf":            OVF,
	"lblc":           LBLC,
	"lblcr":          LBLCR,
	"sh":             SH,
	"dh":             DH,
	"sed":            SED,
	"nq":             NQ,
	"NAT":            NAT,
	"DR":             DR,
	"TUN":            TUN,
	"persistence_timeout": PERSISTENCE_TIMEOUT,
	"protocol":            PROTOCOL,
	"TCP":                 TCP,
	"UDP":                 UDP,
	"sorry_server":        SORRY_SERVER,
	"real_server":         REAL_SERVER,
	"weight":              WEIGHT,
	"inhibit_on_failure":  INHIBIT_ON_FAILURE,
	"TCP_CHECK":           TCP_CHECK,
	"HTTP_GET":            HTTP_GET,
	"SSL_GET":             SSL_GET,
	"SMTP_CHECK":          SMTP_CHECK,
	"DNS_CHECK":           DNS_CHECK,
	"MISC_CHECK":          MISC_CHECK,
	"url":                 URL,
	"path":                PATH,
	"digest":              DIGEST,
	"status_code":         STATUS_CODE,
	"connect_timeout":     CONNECT_TIMEOUT,
	"connect_port":        CONNECT_PORT,
	"connect_ip":          CONNECT_IP,
	"bindto":              BINDTO,
	"bind_port":           BIND_PORT,
	"retry":               RETRY,
	"helo_name":           HELO_NAME,
	"delay_before_retry":  DELAY_BEFORE_RETRY,
	"type":                TYPE,
	"name":                NAME,
	"misc_path":           MISC_PATH,
	"misc_timeout":        MISC_TIMEOUT,
	"warmup":              WARMUP,
	"misc_dynamic":        MISC_DYNAMIC,
	"nb_get_retry":        NB_GET_RETRY,
	"virtualhost":         VIRTUALHOST,
	"alpha":               ALPHA,
	"omega":               OMEGA,
	"quorum":              QUORUM,
	"hysteresis":          HYSTERESIS,
	"quorum_up":           QUORUM_UP,
	"quorum_down":         QUORUM_DOWN,
}

type Lexer struct {
	scanner  scanner.Scanner
	tokens   []int
	pos      int
	filename string
	e        error
}

type Error struct {
	Message  string
	Filename string
	Line     int
	Column   int
}

func (e *Error) Error() string {
	return e.Message
}

func NewLexer(src io.Reader, filename string) *Lexer {
	var lex Lexer
	lex.scanner.Init(src)
	lex.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments
	lex.scanner.IsIdentRune = isIdentRune
	lex.tokens = []int{}
	lex.filename = filename
	return &lex
}

func isIdentRune(ch rune, i int) bool {
	return ch == '_' || ch == '.' || ch == '/' || ch == ':' || ch == '-' || ch == '+' || ch == '*' || ch == '?' || ch == '=' || ch == '&' || ch == '@' || unicode.IsLetter(ch) || unicode.IsDigit(ch)
}

func (l *Lexer) scanNextToken() (int, string) {
	token := int(l.scanner.Scan())
	s := l.scanner.TokenText()

	for s == "!" || s == "#" {
		l.skipComments()

		token = int(l.scanner.Scan())
		s = l.scanner.TokenText()
	}

	log.Debugf("token text: %s\n", s)

	return token, s
}

func (l *Lexer) skipComments() {
	ch := l.scanner.Next()
	for ch != '\n' && ch >= 0 {
		ch = l.scanner.Next()
	}
}

func (l *Lexer) scanInclude(rawfilename string) error {
	curDir, err := filepath.Abs(".")
	if err != nil {
		return err
	}

	baseDir := filepath.Dir(l.filename)
	os.Chdir(baseDir)
	defer os.Chdir(curDir)

	rawpaths, err := filepath.Glob(rawfilename)
	if err != nil {
		return err
	}

	if len(rawpaths) < 1 {
		log.Infof("warning: %s: No such file or directory", rawfilename)
	}

	for _, rawpath := range rawpaths {
		relpath := filepath.Join(baseDir, rawpath)
		l.filename = relpath
		log.Verbosef("--> Parsing ... %s\n", relpath)

		f, err := os.Open(rawpath)
		if err != nil {
			return err
		}

		l.tokenize()

		f.Close()
	}

	return nil
}

func (l *Lexer) Lex(lval *yySymType) int {
	if len(l.tokens) == l.pos {
		return EOF
	}
	token := l.tokens[l.pos]
	l.pos++
	return token
}

func (l *Lexer) tokenize() {
	for {
		token, s := l.scanNextToken()

		for s == "include" {
			token, s = l.scanNextToken()

			if err := l.scanInclude(s); err != nil {
				l.Error(err.Error())
			}

			token, s = l.scanNextToken()
		}

		if token == scanner.EOF {
			break
		}

		if token == scanner.Ident || token == scanner.String {
			token = STRING
		}

		if _, err := strconv.Atoi(s); err == nil {
			token = NUMBER
		}

		if ip := net.ParseIP(s); ip != nil {
			if ip.To4() != nil {
				token = IPV4
			} else if ip.To16() != nil {
				token = IPV6
			} else {
				log.Infof("warning: %s may be IP address?", s)
			}
		}

		if _, _, err := net.ParseCIDR(s); err == nil {
			token = IP_CIDR
		}

		// IPADDR_RANGE(XXX.YYY.ZZZ.WWW-VVV)
		if ss := strings.Split(s, "-"); len(ss) == 2 {
			if net.ParseIP(ss[0]) != nil {
				if ok, _ := regexp.MatchString(`^[\d]{1,3}$`, ss[1]); ok {
					token = IPADDR_RANGE
				}
			}
		}

		if ok, _ := regexp.MatchString(`^[[:xdigit:]]{32}$`, s); ok {
			token = HEX32
		}

		if ok, _ := regexp.MatchString(`/^([[:alnum:]./-_])*`, s); ok {
			token = PATHSTR
		}

		if _, err := mail.ParseAddress(s); err == nil {
			token = EMAIL
		}

		if _, ok := SYMBOL_TABLES[s]; ok {
			token = SYMBOL_TABLES[s]
		}

		l.tokens = append(l.tokens, token)
	}
}

func (l *Lexer) Error(msg string) {
	l.e = &Error{
		Filename: l.filename,
		Line:     l.scanner.Line,
		Column:   l.scanner.Column,
		Message:  msg,
	}
}

func Parse(src io.Reader, filename string) error {
	yyErrorVerbose = true
	l := NewLexer(src, filename)
	l.tokenize()
	if ret := yyParse(l); ret != 0 {
		return l.e
	}
	return l.e
}
<commit_msg>Forget scanner initialization<commit_after>package parser

import (
	"io"
	"net"
	"net/mail"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"text/scanner"
	"unicode"

	"github.com/yuuki/gokc/log"
)

const (
	EOF = 0
)

var SYMBOL_TABLES = map[string]int{
	"{": LB,
	"}": RB,

	"global_defs":                     GLOBALDEFS,
	"notification_email":              NOTIFICATION_EMAIL,
	"notification_email_from":         NOTIFICATION_EMAIL_FROM,
	"smtp_server":                     SMTP_SERVER,
	"smtp_connect_timeout":            SMTP_CONNECT_TIMEOUT,
	"router_id":                       ROUTER_ID,
	"lvs_id":                          LVS_ID,
	"vrrp_mcast_group4":               VRRP_MCAST_GROUP4,
	"vrrp_mcast_group6":               VRRP_MCAST_GROUP6,
	"vrrp_garp_master_delay":          VRRP_GARP_MASTER_DELAY,
	"vrrp_garp_master_repeat":         VRRP_GARP_MASTER_REPEAT,
	"vrrp_garp_master_refresh":        VRRP_GARP_MASTER_REFRESH,
	"vrrp_garp_master_refresh_repeat": VRRP_GARP_MASTER_REFRESH_REPEAT,
	"vrrp_version":                    VRRP_VERSION,

	"static_ipaddress": STATIC_IPADDRESS,
	"static_routes":    STATIC_ROUTES,
	"static_rules":     STATIC_RULES,

	"vrrp_sync_group": VRRP_SYNC_GROUP,
	"group":           GROUP,

	"vrrp_instance":              VRRP_INSTANCE,
	"use_vmac":                   USE_VMAC,
	"version":                    VERSION,
	"vmac_xmit_base":             VMAC_XMIT_BASE,
	"native_ipv6":                NATIVE_IPV6,
	"interface":                  INTERFACE,
	"mcast_src_ip":               MCAST_SRC_IP,
	"unicast_src_ip":             UNICAST_SRC_IP,
	"unicast_peer":               UNICAST_PEER,
	"lvs_sync_daemon_interface":  LVS_SYNC_DAEMON_INTERFACE,
	"virtual_router_id":          VIRTUAL_ROUTER_ID,
	"nopreempt":                  NOPREEMPT,
	"priority":                   PRIORITY,
	"advert_int":                 ADVERT_INT,
	"virtual_ipaddress":          VIRTUAL_IPADDRESS,
	"virtual_ipaddress_excluded": VIRTUAL_IPADDRESS_EXCLUDED,
	"virtual_routes":             VIRTUAL_ROUTES,
	"state":                      STATE,
	"MASTER":                     MASTER,
	"BACKUP":                     BACKUP,
	"garp_master_delay":          GARP_MASTER_DELAY,
	"smtp_alert":                 SMTP_ALERT,
	"authentication":             AUTHENTICATION,
	"auth_type":                  AUTH_TYPE,
	"auth_pass":                  AUTH_PASS,
	"PASS":                       PASS,
	"AH":                         AH,
	"label":                      LABEL,
	"dev":                        DEV,
	"scope":                      SCOPE,
	"site":                       SITE,
	"link":                       LINK,
	"host":                       HOST,
	"nowhere":                    NOWHERE,
	"global":                     GLOBAL,
	"brd":                        BRD,
	"src":                        SRC,
	"from":                       FROM,
	"to":                         TO,
	"via":                        VIA,
	"gw":                         GW,
	"or":                         OR,
	"table":                      TABLE,
	"metric":                     METRIC,
	"blackhole":                  BLACKHOLE,
	"track_interface":            TRACK_INTERFACE,
	"track_script":               TRACK_SCRIPT,
	"dont_track_primary":         DONT_TRACK_PRIMARY,
	"notify_master":              NOTIFY_MASTER,
	"notify_backup":              NOTIFY_BACKUP,
	"notify_fault":               NOTIFY_FAULT,
	"notify_stop":                NOTIFY_STOP,
	"notify":                     NOTIFY,

	"vrrp_script": VRRP_SCRIPT,
	"script":      SCRIPT,
	"interval":    INTERVAL,
	"timeout":     TIMEOUT,
	"fall":        FALL,
	"rise":        RISE,

	"virtual_server_group": VIRTUAL_SERVER_GROUP,
	"fwmark":               FWMARK,

	"virtual_server": VIRTUAL_SERVER,
	"delay_loop":     DELAY_LOOP,
	"lb_algo":        LB_ALGO,
	"lb_kind":        LB_KIND,
	"lvs_sched":      LVS_SCHED,
	"lvs_method":     LVS_METHOD,
	"rr":             RR,
	"wrr":            WRR,
	"lc":             LC,
	"wlc":            WLC,
	"fo":             FO,
	"ovf":            OVF,
	"lblc":           LBLC,
	"lblcr":          LBLCR,
	"sh":             SH,
	"dh":             DH,
	"sed":            SED,
	"nq":             NQ,
	"NAT":            NAT,
	"DR":             DR,
	"TUN":            TUN,
	"persistence_timeout": PERSISTENCE_TIMEOUT,
	"protocol":            PROTOCOL,
	"TCP":                 TCP,
	"UDP":                 UDP,
	"sorry_server":        SORRY_SERVER,
	"real_server":         REAL_SERVER,
	"weight":              WEIGHT,
	"inhibit_on_failure":  INHIBIT_ON_FAILURE,
	"TCP_CHECK":           TCP_CHECK,
	"HTTP_GET":            HTTP_GET,
	"SSL_GET":             SSL_GET,
	"SMTP_CHECK":          SMTP_CHECK,
	"DNS_CHECK":           DNS_CHECK,
	"MISC_CHECK":          MISC_CHECK,
	"url":                 URL,
	"path":                PATH,
	"digest":              DIGEST,
	"status_code":         STATUS_CODE,
	"connect_timeout":     CONNECT_TIMEOUT,
	"connect_port":        CONNECT_PORT,
	"connect_ip":          CONNECT_IP,
	"bindto":              BINDTO,
	"bind_port":           BIND_PORT,
	"retry":               RETRY,
	"helo_name":           HELO_NAME,
	"delay_before_retry":  DELAY_BEFORE_RETRY,
	"type":                TYPE,
	"name":                NAME,
	"misc_path":           MISC_PATH,
	"misc_timeout":        MISC_TIMEOUT,
	"warmup":              WARMUP,
	"misc_dynamic":        MISC_DYNAMIC,
	"nb_get_retry":        NB_GET_RETRY,
	"virtualhost":         VIRTUALHOST,
	"alpha":               ALPHA,
	"omega":               OMEGA,
	"quorum":              QUORUM,
	"hysteresis":          HYSTERESIS,
	"quorum_up":           QUORUM_UP,
	"quorum_down":         QUORUM_DOWN,
}

type Lexer struct {
	scanner     scanner.Scanner
	tokens      []int
	pos         int
	filename    string
	curFilename string
	e           error
}

type Error struct {
	Message  string
	Filename string
	Line     int
	Column   int
}

func (e *Error) Error() string {
	return e.Message
}

func NewLexer(src io.Reader, filename string) *Lexer {
	var lex Lexer
	lex.scanner.Init(src)
	lex.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments
	lex.scanner.IsIdentRune = isIdentRune
	lex.tokens = []int{}
	lex.filename = filename
	lex.curFilename = filename
	return &lex
}

func isIdentRune(ch rune, i int) bool {
	return ch == '_' || ch == '.' || ch == '/' || ch == ':' || ch == '-' || ch == '+' || ch == '*' || ch == '?' || ch == '=' || ch == '&' || ch == '@' || unicode.IsLetter(ch) || unicode.IsDigit(ch)
}

func (l *Lexer) scanNextToken() (int, string) {
	token := int(l.scanner.Scan())
	s := l.scanner.TokenText()

	for s == "!" || s == "#" {
		l.skipComments()

		token = int(l.scanner.Scan())
		s = l.scanner.TokenText()
	}

	log.Debugf("token text: %s\n", s)

	return token, s
}

func (l *Lexer) skipComments() {
	ch := l.scanner.Next()
	for ch != '\n' && ch >= 0 {
		ch = l.scanner.Next()
	}
}

func (l *Lexer) scanInclude(rawfilename string) error {
	curDir, err := filepath.Abs(".")
	if err != nil {
		return err
	}

	baseDir := filepath.Dir(l.filename)
	os.Chdir(baseDir)
	defer os.Chdir(curDir)

	rawpaths, err := filepath.Glob(filepath.Join(baseDir, rawfilename))
	if err != nil {
		return err
	}

	if len(rawpaths) < 1 {
		log.Infof("warning: %s: No such file or directory", rawfilename)
	}

	prevScanner := l.scanner
	defer func() { l.scanner = prevScanner }()

	for _, rawpath := range rawpaths {
		l.curFilename = rawpath
		log.Verbosef("--> Parsing ... %s\n", rawpath)

		f, err := os.Open(rawpath)
		if err != nil {
			return err
		}

		l.scanner.Init(f)
		l.scanner.Mode &^= scanner.ScanInts | scanner.ScanFloats | scanner.ScanChars | scanner.ScanRawStrings | scanner.ScanComments | scanner.SkipComments
		l.scanner.IsIdentRune = isIdentRune
		l.tokenize()

		f.Close()
	}

	return nil
}

func (l *Lexer) Lex(lval *yySymType) int {
	if len(l.tokens) == l.pos {
		return EOF
	}
	token := l.tokens[l.pos]
	l.pos++
	return token
}

func (l *Lexer) tokenize() {
	for {
		token, s := l.scanNextToken()

		for s == "include" {
			token, s = l.scanNextToken()

			if err := l.scanInclude(s); err != nil {
				l.Error(err.Error())
			}

			token, s = l.scanNextToken()
		}

		if token == scanner.EOF {
			break
		}

		if token == scanner.Ident || token == scanner.String {
			token = STRING
		}

		if _, err := strconv.Atoi(s); err == nil {
			token = NUMBER
		}

		if ip := net.ParseIP(s); ip != nil {
			if ip.To4() != nil {
				token = IPV4
			} else if ip.To16() != nil {
				token = IPV6
			} else {
				log.Infof("warning: %s may be IP address?", s)
			}
		}

		if _, _, err := net.ParseCIDR(s); err == nil {
			token = IP_CIDR
		}

		// IPADDR_RANGE(XXX.YYY.ZZZ.WWW-VVV)
		if ss := strings.Split(s, "-"); len(ss) == 2 {
			if net.ParseIP(ss[0]) != nil {
				if ok, _ := regexp.MatchString(`^[\d]{1,3}$`, ss[1]); ok {
					token = IPADDR_RANGE
				}
			}
		}

		if ok, _ := regexp.MatchString(`^[[:xdigit:]]{32}$`, s); ok {
			token = HEX32
		}

		if ok, _ := regexp.MatchString(`/^([[:alnum:]./-_])*`, s); ok {
			token = PATHSTR
		}

		if _, err := mail.ParseAddress(s); err == nil {
			token = EMAIL
		}

		if _, ok := SYMBOL_TABLES[s]; ok {
			token = SYMBOL_TABLES[s]
		}

		l.tokens = append(l.tokens, token)
	}
}

func (l *Lexer) Error(msg string) {
	l.e = &Error{
		Filename: l.curFilename,
		Line:     l.scanner.Line,
		Column:   l.scanner.Column,
		Message:  msg,
	}
}

func Parse(src io.Reader, filename string) error {
	yyErrorVerbose = true
	l := NewLexer(src, filename)
	l.tokenize()
	if ret := yyParse(l); ret != 0 {
		return l.e
	}
	return l.e
}
<|endoftext|>