<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/gorilla/mux"
	"github.com/gorilla/websocket"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

type commandPostContent struct {
	Command     []string          `json:"command"`
	WaitForWS   bool              `json:"wait-for-websocket"`
	Interactive bool              `json:"interactive"`
	Environment map[string]string `json:"environment"`
	Width       int               `json:"width"`
	Height      int               `json:"height"`
}

type execWs struct {
	command   []string
	container container
	env       map[string]string

	rootUid          int
	rootGid          int
	conns            map[int]*websocket.Conn
	connsLock        sync.Mutex
	allConnected     chan bool
	controlConnected chan bool
	interactive      bool
	fds              map[int]string
	width            int
	height           int
}

func (s *execWs) Metadata() interface{} {
	fds := shared.Jmap{}
	for fd, secret := range s.fds {
		if fd == -1 {
			fds["control"] = secret
		} else {
			fds[strconv.Itoa(fd)] = secret
		}
	}

	return shared.Jmap{"fds": fds}
}

func (s *execWs) Connect(op *operation, r *http.Request, w http.ResponseWriter) error {
	secret := r.FormValue("secret")
	if secret == "" {
		return fmt.Errorf("missing secret")
	}

	for fd, fdSecret := range s.fds {
		if secret == fdSecret {
			conn, err := shared.WebsocketUpgrader.Upgrade(w, r, nil)
			if err != nil {
				return err
			}

			s.connsLock.Lock()
			s.conns[fd] = conn
			s.connsLock.Unlock()

			if fd == -1 {
				s.controlConnected <- true
				return nil
			}

			for i, c := range s.conns {
				if i != -1 && c == nil {
					return nil
				}
			}
			s.allConnected <- true
			return nil
		}
	}

	/* If we didn't find the right secret, the user provided a bad one,
	 * which 403, not 404, since this operation actually exists */
	return os.ErrPermission
}

func (s *execWs) Do(op *operation) error {
	<-s.allConnected

	var err error
	var ttys []*os.File
	var ptys []*os.File

	var stdin *os.File
	var stdout *os.File
	var stderr *os.File

	if s.interactive {
		ttys = make([]*os.File, 1)
		ptys = make([]*os.File, 1)
		ptys[0], ttys[0], err = shared.OpenPty(s.rootUid, s.rootGid)

		stdin = ttys[0]
		stdout = ttys[0]
		stderr = ttys[0]

		if s.width > 0 && s.height > 0 {
			shared.SetSize(int(ptys[0].Fd()), s.width, s.height)
		}
	} else {
		ttys = make([]*os.File, 3)
		ptys = make([]*os.File, 3)
		for i := 0; i < len(ttys); i++ {
			ptys[i], ttys[i], err = shared.Pipe()
			if err != nil {
				return err
			}
		}

		stdin = ptys[0]
		stdout = ttys[1]
		stderr = ttys[2]
	}

	controlExit := make(chan bool)
	var wgEOF sync.WaitGroup

	if s.interactive {
		wgEOF.Add(1)
		go func() {
			select {
			case <-s.controlConnected:
				break

			case <-controlExit:
				return
			}

			for {
				mt, r, err := s.conns[-1].NextReader()
				if mt == websocket.CloseMessage {
					break
				}

				if err != nil {
					shared.LogDebugf("Got error getting next reader %s", err)
					break
				}

				buf, err := ioutil.ReadAll(r)
				if err != nil {
					shared.LogDebugf("Failed to read message %s", err)
					break
				}

				command := shared.ContainerExecControl{}

				if err := json.Unmarshal(buf, &command); err != nil {
					shared.LogDebugf("Failed to unmarshal control socket command: %s", err)
					continue
				}

				if command.Command == "window-resize" {
					winchWidth, err := strconv.Atoi(command.Args["width"])
					if err != nil {
						shared.LogDebugf("Unable to extract window width: %s", err)
						continue
					}

					winchHeight, err := strconv.Atoi(command.Args["height"])
					if err != nil {
						shared.LogDebugf("Unable to extract window height: %s", err)
						continue
					}

					err = shared.SetSize(int(ptys[0].Fd()), winchWidth, winchHeight)
					if err != nil {
						shared.LogDebugf("Failed to set window size to: %dx%d", winchWidth, winchHeight)
						continue
					}
				}
			}
		}()
		go func() {
			readDone, writeDone := shared.WebsocketMirror(s.conns[0], ptys[0], ptys[0])
			<-readDone
			<-writeDone
			s.conns[0].Close()
			wgEOF.Done()
		}()
	} else {
		wgEOF.Add(len(ttys) - 1)
		for i := 0; i < len(ttys); i++ {
			go func(i int) {
				if i == 0 {
					<-shared.WebsocketRecvStream(ttys[i], s.conns[i])
					ttys[i].Close()
				} else {
					<-shared.WebsocketSendStream(s.conns[i], ptys[i], -1)
					ptys[i].Close()
					wgEOF.Done()
				}
			}(i)
		}
	}

	cmdResult, cmdErr := s.container.Exec(s.command, s.env, stdin, stdout, stderr)

	for _, tty := range ttys {
		tty.Close()
	}

	if s.conns[-1] == nil {
		if s.interactive {
			controlExit <- true
		}
	} else {
		s.conns[-1].Close()
	}

	wgEOF.Wait()

	for _, pty := range ptys {
		pty.Close()
	}

	metadata := shared.Jmap{"return": cmdResult}
	err = op.UpdateMetadata(metadata)
	if err != nil {
		return err
	}

	return cmdErr
}

func containerExecPost(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]
	c, err := containerLoadByName(d, name)
	if err != nil {
		return SmartError(err)
	}

	if !c.IsRunning() {
		return BadRequest(fmt.Errorf("Container is not running."))
	}

	if c.IsFrozen() {
		return BadRequest(fmt.Errorf("Container is frozen."))
	}

	post := commandPostContent{}
	buf, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return BadRequest(err)
	}

	if err := json.Unmarshal(buf, &post); err != nil {
		return BadRequest(err)
	}

	env := map[string]string{}

	for k, v := range c.ExpandedConfig() {
		if strings.HasPrefix(k, "environment.") {
			env[strings.TrimPrefix(k, "environment.")] = v
		}
	}

	if post.Environment != nil {
		for k, v := range post.Environment {
			env[k] = v
		}
	}

	if post.WaitForWS {
		ws := &execWs{}
		ws.fds = map[int]string{}
		idmapset := c.IdmapSet()
		if idmapset != nil {
			ws.rootUid, ws.rootGid = idmapset.ShiftIntoNs(0, 0)
		}
		ws.conns = map[int]*websocket.Conn{}
		ws.conns[-1] = nil
		ws.conns[0] = nil
		if !post.Interactive {
			ws.conns[1] = nil
			ws.conns[2] = nil
		}
		ws.allConnected = make(chan bool, 1)
		ws.controlConnected = make(chan bool, 1)
		ws.interactive = post.Interactive
		for i := -1; i < len(ws.conns)-1; i++ {
			ws.fds[i], err = shared.RandomCryptoString()
			if err != nil {
				return InternalError(err)
			}
		}

		ws.command = post.Command
		ws.container = c
		ws.env = env

		ws.width = post.Width
		ws.height = post.Height

		resources := map[string][]string{}
		resources["containers"] = []string{ws.container.Name()}

		op, err := operationCreate(operationClassWebsocket, resources, ws.Metadata(), ws.Do, nil, ws.Connect)
		if err != nil {
			return InternalError(err)
		}

		return OperationResponse(op)
	}

	run := func(op *operation) error {
		nullDev, err := os.OpenFile(os.DevNull, os.O_RDWR, 0666)
		if err != nil {
			return err
		}
		defer nullDev.Close()

		cmdResult, cmdErr := c.Exec(post.Command, env, nil, nil, nil)
		metadata := shared.Jmap{"return": cmdResult}
		err = op.UpdateMetadata(metadata)
		if err != nil {
			shared.LogError("error updating metadata for cmd", log.Ctx{"err": err, "cmd": post.Command})
		}
		return cmdErr
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}

	op, err := operationCreate(operationClassTask, resources, nil, run, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}
<commit_msg>snappy: Add /snap/bin to PATH if present<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/gorilla/mux"
	"github.com/gorilla/websocket"

	"github.com/lxc/lxd/shared"

	log "gopkg.in/inconshreveable/log15.v2"
)

type commandPostContent struct {
	Command     []string          `json:"command"`
	WaitForWS   bool              `json:"wait-for-websocket"`
	Interactive bool              `json:"interactive"`
	Environment map[string]string `json:"environment"`
	Width       int               `json:"width"`
	Height      int               `json:"height"`
}

type execWs struct {
	command   []string
	container container
	env       map[string]string

	rootUid          int
	rootGid          int
	conns            map[int]*websocket.Conn
	connsLock        sync.Mutex
	allConnected     chan bool
	controlConnected chan bool
	interactive      bool
	fds              map[int]string
	width            int
	height           int
}

func (s *execWs) Metadata() interface{} {
	fds := shared.Jmap{}
	for fd, secret := range s.fds {
		if fd == -1 {
			fds["control"] = secret
		} else {
			fds[strconv.Itoa(fd)] = secret
		}
	}

	return shared.Jmap{"fds": fds}
}

func (s *execWs) Connect(op *operation, r *http.Request, w http.ResponseWriter) error {
	secret := r.FormValue("secret")
	if secret == "" {
		return fmt.Errorf("missing secret")
	}

	for fd, fdSecret := range s.fds {
		if secret == fdSecret {
			conn, err := shared.WebsocketUpgrader.Upgrade(w, r, nil)
			if err != nil {
				return err
			}

			s.connsLock.Lock()
			s.conns[fd] = conn
			s.connsLock.Unlock()

			if fd == -1 {
				s.controlConnected <- true
				return nil
			}

			for i, c := range s.conns {
				if i != -1 && c == nil {
					return nil
				}
			}
			s.allConnected <- true
			return nil
		}
	}

	/* If we didn't find the right secret, the user provided a bad one,
	 * which 403, not 404, since this operation actually exists */
	return os.ErrPermission
}

func (s *execWs) Do(op *operation) error {
	<-s.allConnected

	var err error
	var ttys []*os.File
	var ptys []*os.File

	var stdin *os.File
	var stdout *os.File
	var stderr *os.File

	if s.interactive {
		ttys = make([]*os.File, 1)
		ptys = make([]*os.File, 1)
		ptys[0], ttys[0], err = shared.OpenPty(s.rootUid, s.rootGid)

		stdin = ttys[0]
		stdout = ttys[0]
		stderr = ttys[0]

		if s.width > 0 && s.height > 0 {
			shared.SetSize(int(ptys[0].Fd()), s.width, s.height)
		}
	} else {
		ttys = make([]*os.File, 3)
		ptys = make([]*os.File, 3)
		for i := 0; i < len(ttys); i++ {
			ptys[i], ttys[i], err = shared.Pipe()
			if err != nil {
				return err
			}
		}

		stdin = ptys[0]
		stdout = ttys[1]
		stderr = ttys[2]
	}

	controlExit := make(chan bool)
	var wgEOF sync.WaitGroup

	if s.interactive {
		wgEOF.Add(1)
		go func() {
			select {
			case <-s.controlConnected:
				break

			case <-controlExit:
				return
			}

			for {
				mt, r, err := s.conns[-1].NextReader()
				if mt == websocket.CloseMessage {
					break
				}

				if err != nil {
					shared.LogDebugf("Got error getting next reader %s", err)
					break
				}

				buf, err := ioutil.ReadAll(r)
				if err != nil {
					shared.LogDebugf("Failed to read message %s", err)
					break
				}

				command := shared.ContainerExecControl{}

				if err := json.Unmarshal(buf, &command); err != nil {
					shared.LogDebugf("Failed to unmarshal control socket command: %s", err)
					continue
				}

				if command.Command == "window-resize" {
					winchWidth, err := strconv.Atoi(command.Args["width"])
					if err != nil {
						shared.LogDebugf("Unable to extract window width: %s", err)
						continue
					}

					winchHeight, err := strconv.Atoi(command.Args["height"])
					if err != nil {
						shared.LogDebugf("Unable to extract window height: %s", err)
						continue
					}

					err = shared.SetSize(int(ptys[0].Fd()), winchWidth, winchHeight)
					if err != nil {
						shared.LogDebugf("Failed to set window size to: %dx%d", winchWidth, winchHeight)
						continue
					}
				}
			}
		}()
		go func() {
			readDone, writeDone := shared.WebsocketMirror(s.conns[0], ptys[0], ptys[0])
			<-readDone
			<-writeDone
			s.conns[0].Close()
			wgEOF.Done()
		}()
	} else {
		wgEOF.Add(len(ttys) - 1)
		for i := 0; i < len(ttys); i++ {
			go func(i int) {
				if i == 0 {
					<-shared.WebsocketRecvStream(ttys[i], s.conns[i])
					ttys[i].Close()
				} else {
					<-shared.WebsocketSendStream(s.conns[i], ptys[i], -1)
					ptys[i].Close()
					wgEOF.Done()
				}
			}(i)
		}
	}

	cmdResult, cmdErr := s.container.Exec(s.command, s.env, stdin, stdout, stderr)

	for _, tty := range ttys {
		tty.Close()
	}

	if s.conns[-1] == nil {
		if s.interactive {
			controlExit <- true
		}
	} else {
		s.conns[-1].Close()
	}

	wgEOF.Wait()

	for _, pty := range ptys {
		pty.Close()
	}

	metadata := shared.Jmap{"return": cmdResult}
	err = op.UpdateMetadata(metadata)
	if err != nil {
		return err
	}

	return cmdErr
}

func containerExecPost(d *Daemon, r *http.Request) Response {
	name := mux.Vars(r)["name"]
	c, err := containerLoadByName(d, name)
	if err != nil {
		return SmartError(err)
	}

	if !c.IsRunning() {
		return BadRequest(fmt.Errorf("Container is not running."))
	}

	if c.IsFrozen() {
		return BadRequest(fmt.Errorf("Container is frozen."))
	}

	post := commandPostContent{}
	buf, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return BadRequest(err)
	}

	if err := json.Unmarshal(buf, &post); err != nil {
		return BadRequest(err)
	}

	env := map[string]string{}

	for k, v := range c.ExpandedConfig() {
		if strings.HasPrefix(k, "environment.") {
			env[strings.TrimPrefix(k, "environment.")] = v
		}
	}

	if post.Environment != nil {
		for k, v := range post.Environment {
			env[k] = v
		}
	}

	_, ok := env["PATH"]
	if !ok {
		env["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
		if shared.PathExists(fmt.Sprintf("%s/snap/bin", c.RootfsPath())) {
			env["PATH"] = fmt.Sprintf("%s:/snap/bin", env["PATH"])
		}
	}

	if post.WaitForWS {
		ws := &execWs{}
		ws.fds = map[int]string{}
		idmapset := c.IdmapSet()
		if idmapset != nil {
			ws.rootUid, ws.rootGid = idmapset.ShiftIntoNs(0, 0)
		}
		ws.conns = map[int]*websocket.Conn{}
		ws.conns[-1] = nil
		ws.conns[0] = nil
		if !post.Interactive {
			ws.conns[1] = nil
			ws.conns[2] = nil
		}
		ws.allConnected = make(chan bool, 1)
		ws.controlConnected = make(chan bool, 1)
		ws.interactive = post.Interactive
		for i := -1; i < len(ws.conns)-1; i++ {
			ws.fds[i], err = shared.RandomCryptoString()
			if err != nil {
				return InternalError(err)
			}
		}

		ws.command = post.Command
		ws.container = c
		ws.env = env

		ws.width = post.Width
		ws.height = post.Height

		resources := map[string][]string{}
		resources["containers"] = []string{ws.container.Name()}

		op, err := operationCreate(operationClassWebsocket, resources, ws.Metadata(), ws.Do, nil, ws.Connect)
		if err != nil {
			return InternalError(err)
		}

		return OperationResponse(op)
	}

	run := func(op *operation) error {
		nullDev, err := os.OpenFile(os.DevNull, os.O_RDWR, 0666)
		if err != nil {
			return err
		}
		defer nullDev.Close()

		cmdResult, cmdErr := c.Exec(post.Command, env, nil, nil, nil)
		metadata := shared.Jmap{"return": cmdResult}
		err = op.UpdateMetadata(metadata)
		if err != nil {
			shared.LogError("error updating metadata for cmd", log.Ctx{"err": err, "cmd": post.Command})
		}
		return cmdErr
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}

	op, err := operationCreate(operationClassTask, resources, nil, run, nil, nil)
	if err != nil {
		return InternalError(err)
	}

	return OperationResponse(op)
}
<|endoftext|>


<commit_before>package core_test

import (
  "github.com/orfjackal/gospec/src/gospec"
  "math/rand"
  "testing"
)

// List of all specs here
func TestAllSpecs(t *testing.T) {
  r := gospec.NewRunner()
  // r.AddSpec(ZBoxSpec)
  // r.AddSpec(ZBoxReverseSpec)
  // r.AddSpec(LongestSuffixAsPrefixSpec)
  // r.AddSpec(BoyerMooreSpec)
  r.AddSpec(BoyerMooreReaderSpec)
  // r.AddSpec(AhoCorasickSpec)
  // r.AddSpec(AhoCorasickReaderSpec)
  gospec.MainGoTest(r, t)
}

// The rest of this file is utility functions for testing

// Returns a string of length n of all the same character, c
func makeTestString1(n int, c byte) []byte {
  b := make([]byte, n)
  for i := range b {
    b[i] = c
  }
  return b
}

// Returns a string of length n, first half one character, second half a
// different character
func makeTestString2(n int) []byte {
  b := make([]byte, n)
  for i := n / 2; i < n; i++ {
    b[i] = 1
  }
  return b
}

// Returns a string of length n, cycling through the number 0-(r-1)
func makeTestString3(n, r int) []byte {
  b := make([]byte, n)
  for i := range b {
    b[i] = byte(i % r)
  }
  return b
}

// Returns a string of length n consisting of random characters less than r,
// and using seed s
func makeTestString4(n, r, s int) []byte {
  rand.Seed(int64(s))
  b := make([]byte, n)
  for i := range b {
    b[i] = byte(rand.Intn(256) % r)
  }
  return b
}

func augment(b []byte, radix int) bool {
  for i := range b {
    if int(b[i]) < radix-1 {
      b[i]++
      return true
    } else {
      b[i] = 0
    }
  }
  return false
}
<commit_msg>Turned back on a bunch of tests<commit_after>package core_test

import (
  "github.com/orfjackal/gospec/src/gospec"
  "math/rand"
  "testing"
)

// List of all specs here
func TestAllSpecs(t *testing.T) {
  r := gospec.NewRunner()
  r.AddSpec(ZBoxSpec)
  r.AddSpec(ZBoxReverseSpec)
  r.AddSpec(LongestSuffixAsPrefixSpec)
  r.AddSpec(BoyerMooreSpec)
  r.AddSpec(BoyerMooreReaderSpec)
  r.AddSpec(AhoCorasickSpec)
  r.AddSpec(AhoCorasickReaderSpec)
  gospec.MainGoTest(r, t)
}

// The rest of this file is utility functions for testing

// Returns a string of length n of all the same character, c
func makeTestString1(n int, c byte) []byte {
  b := make([]byte, n)
  for i := range b {
    b[i] = c
  }
  return b
}

// Returns a string of length n, first half one character, second half a
// different character
func makeTestString2(n int) []byte {
  b := make([]byte, n)
  for i := n / 2; i < n; i++ {
    b[i] = 1
  }
  return b
}

// Returns a string of length n, cycling through the number 0-(r-1)
func makeTestString3(n, r int) []byte {
  b := make([]byte, n)
  for i := range b {
    b[i] = byte(i % r)
  }
  return b
}

// Returns a string of length n consisting of random characters less than r,
// and using seed s
func makeTestString4(n, r, s int) []byte {
  rand.Seed(int64(s))
  b := make([]byte, n)
  for i := range b {
    b[i] = byte(rand.Intn(256) % r)
  }
  return b
}

func augment(b []byte, radix int) bool {
  for i := range b {
    if int(b[i]) < radix-1 {
      b[i]++
      return true
    } else {
      b[i] = 0
    }
  }
  return false
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logs

import (
	"flag"
	"fmt"
	"strings"

	"github.com/go-logr/logr"
	"github.com/spf13/pflag"

	"k8s.io/component-base/logs/sanitization"
	"k8s.io/klog/v2"
)

const (
	logFormatFlagName = "logging-format"
	defaultLogFormat  = "text"
)

// List of logs (k8s.io/klog + k8s.io/component-base/logs) flags supported by all logging formats
var supportedLogsFlags = map[string]struct{}{
	"v": {},
	// TODO: support vmodule after 1.19 Alpha
}

// Options has klog format parameters
type Options struct {
	LogFormat       string
	LogSanitization bool
}

// NewOptions return new klog options
func NewOptions() *Options {
	return &Options{
		LogFormat: defaultLogFormat,
	}
}

// Validate verifies if any unsupported flag is set
// for non-default logging format
func (o *Options) Validate() []error {
	errs := []error{}
	if o.LogFormat != defaultLogFormat {
		allFlags := unsupportedLoggingFlags()
		for _, fname := range allFlags {
			if flagIsSet(fname) {
				errs = append(errs, fmt.Errorf("non-default logging format doesn't honor flag: %s", fname))
			}
		}
	}
	if _, err := o.Get(); err != nil {
		errs = append(errs, fmt.Errorf("unsupported log format: %s", o.LogFormat))
	}
	return errs
}

func flagIsSet(name string) bool {
	f := flag.Lookup(name)
	if f != nil {
		return f.DefValue != f.Value.String()
	}
	pf := pflag.Lookup(name)
	if pf != nil {
		return pf.DefValue != pf.Value.String()
	}
	panic("failed to lookup unsupported log flag")
}

// AddFlags add logging-format flag
func (o *Options) AddFlags(fs *pflag.FlagSet) {
	unsupportedFlags := fmt.Sprintf("--%s", strings.Join(unsupportedLoggingFlags(), ", --"))
	formats := fmt.Sprintf(`"%s"`, strings.Join(logRegistry.List(), `", "`))
	fs.StringVar(&o.LogFormat, logFormatFlagName, defaultLogFormat, fmt.Sprintf("Sets the log format. Permitted formats: %s.\nNon-default formats don't honor these flags: %s.\nNon-default choices are currently alpha and subject to change without warning.", formats, unsupportedFlags))

	// No new log formats should be added after generation is of flag options
	logRegistry.Freeze()
	fs.BoolVar(&o.LogSanitization, "experimental-logging-sanitization", o.LogSanitization, `[Experimental] When enabled prevents logging of fields tagged as sensitive (passwords, keys, tokens).
Runtime log sanitization may introduce significant computation overhead and therefore should not be enabled in production.`)
}

// Apply set klog logger from LogFormat type
func (o *Options) Apply() {
	// if log format not exists, use nil loggr
	loggr, _ := o.Get()
	klog.SetLogger(loggr)
	if o.LogSanitization {
		klog.SetLogFilter(&sanitization.SanitizingFilter{})
	}
}

// Get logger with LogFormat field
func (o *Options) Get() (logr.Logger, error) {
	return logRegistry.Get(o.LogFormat)
}

func unsupportedLoggingFlags() []string {
	allFlags := []string{}

	// k8s.io/klog flags
	fs := &flag.FlagSet{}
	klog.InitFlags(fs)
	fs.VisitAll(func(flag *flag.Flag) {
		if _, found := supportedLogsFlags[flag.Name]; !found {
			allFlags = append(allFlags, flag.Name)
		}
	})

	// k8s.io/component-base/logs flags
	pfs := &pflag.FlagSet{}
	AddFlags(pfs)
	pfs.VisitAll(func(flag *pflag.Flag) {
		if _, found := supportedLogsFlags[flag.Name]; !found {
			allFlags = append(allFlags, flag.Name)
		}
	})
	return allFlags
}
<commit_msg>Fix unified flag in --logging-format description<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package logs

import (
	"flag"
	"fmt"
	"strings"

	"github.com/go-logr/logr"
	"github.com/spf13/pflag"

	"k8s.io/component-base/logs/sanitization"
	"k8s.io/klog/v2"
)

const (
	logFormatFlagName = "logging-format"
	defaultLogFormat  = "text"
)

// List of logs (k8s.io/klog + k8s.io/component-base/logs) flags supported by all logging formats
var supportedLogsFlags = map[string]struct{}{
	"v": {},
	// TODO: support vmodule after 1.19 Alpha
}

// Options has klog format parameters
type Options struct {
	LogFormat       string
	LogSanitization bool
}

// NewOptions return new klog options
func NewOptions() *Options {
	return &Options{
		LogFormat: defaultLogFormat,
	}
}

// Validate verifies if any unsupported flag is set
// for non-default logging format
func (o *Options) Validate() []error {
	errs := []error{}
	if o.LogFormat != defaultLogFormat {
		allFlags := unsupportedLoggingFlags()
		for _, fname := range allFlags {
			if flagIsSet(fname) {
				errs = append(errs, fmt.Errorf("non-default logging format doesn't honor flag: %s", fname))
			}
		}
	}
	if _, err := o.Get(); err != nil {
		errs = append(errs, fmt.Errorf("unsupported log format: %s", o.LogFormat))
	}
	return errs
}

func flagIsSet(name string) bool {
	f := flag.Lookup(name)
	if f != nil {
		return f.DefValue != f.Value.String()
	}
	pf := pflag.Lookup(name)
	if pf != nil {
		return pf.DefValue != pf.Value.String()
	}
	panic("failed to lookup unsupported log flag")
}

// AddFlags add logging-format flag
func (o *Options) AddFlags(fs *pflag.FlagSet) {
	unsupportedFlags := fmt.Sprintf("--%s", strings.Join(unsupportedLoggingFlags(), ", --"))
	formats := fmt.Sprintf(`"%s"`, strings.Join(logRegistry.List(), `", "`))
	fs.StringVar(&o.LogFormat, logFormatFlagName, defaultLogFormat, fmt.Sprintf("Sets the log format. Permitted formats: %s.\nNon-default formats don't honor these flags: %s.\nNon-default choices are currently alpha and subject to change without warning.", formats, unsupportedFlags))

	// No new log formats should be added after generation is of flag options
	logRegistry.Freeze()
	fs.BoolVar(&o.LogSanitization, "experimental-logging-sanitization", o.LogSanitization, `[Experimental] When enabled prevents logging of fields tagged as sensitive (passwords, keys, tokens).
Runtime log sanitization may introduce significant computation overhead and therefore should not be enabled in production.`)
}

// Apply set klog logger from LogFormat type
func (o *Options) Apply() {
	// if log format not exists, use nil loggr
	loggr, _ := o.Get()
	klog.SetLogger(loggr)
	if o.LogSanitization {
		klog.SetLogFilter(&sanitization.SanitizingFilter{})
	}
}

// Get logger with LogFormat field
func (o *Options) Get() (logr.Logger, error) {
	return logRegistry.Get(o.LogFormat)
}

func unsupportedLoggingFlags() []string {
	allFlags := []string{}

	// k8s.io/klog flags
	fs := &flag.FlagSet{}
	klog.InitFlags(fs)
	fs.VisitAll(func(flag *flag.Flag) {
		if _, found := supportedLogsFlags[flag.Name]; !found {
			allFlags = append(allFlags, strings.Replace(flag.Name, "_", "-", -1))
		}
	})

	// k8s.io/component-base/logs flags
	pfs := &pflag.FlagSet{}
	AddFlags(pfs)
	pfs.VisitAll(func(flag *pflag.Flag) {
		if _, found := supportedLogsFlags[flag.Name]; !found {
			allFlags = append(allFlags, flag.Name)
		}
	})
	return allFlags
}
<|endoftext|>


<commit_before>package mains

import (
	"errors"
	"fmt"
	"strings"

	"../completion"
	"../lua"
	"../readline"
)

var completionHook lua.Pushable = lua.TNil{}

func luaHookForComplete(this *readline.Buffer, rv *completion.List) (*completion.List, error) {
	L, L_ok := this.Context.Value("lua").(lua.Lua)
	if !L_ok {
		return rv, errors.New("listUpComplete: could not get lua instance")
	}

	L.Push(completionHook)
	if L.IsFunction(-1) {
		L.NewTable()
		L.PushString(rv.RawWord)
		L.SetField(-2, "rawword")
		L.Push(rv.Pos + 1)
		L.SetField(-2, "pos")
		L.PushString(rv.AllLine)
		L.SetField(-2, "text")
		L.PushString(rv.Word)
		L.SetField(-2, "word")
		L.NewTable()
		for key, val := range rv.List {
			L.Push(1 + key)
			L.PushString(val.InsertStr)
			L.SetTable(-3)
		}
		L.SetField(-2, "list")
		if err := L.Call(1, 1); err != nil {
			fmt.Println(err)
		}
		if L.IsTable(-1) {
			list := make([]completion.Element, 0, len(rv.List)+32)
			wordUpr := strings.ToUpper(rv.Word)
			for i := 1; true; i++ {
				L.Push(i)
				L.GetTable(-2)
				str, strErr := L.ToString(-1)
				L.Pop(1)
				if strErr != nil || str == "" {
					break
				}
				strUpr := strings.ToUpper(str)
				if strings.HasPrefix(strUpr, wordUpr) {
					list = append(list, completion.Element{str, str})
				}
			}
			if len(list) > 0 {
				rv.List = list
			}
		}
	}
	L.Pop(1) // remove something not function or result-table
	return rv, nil
}
<commit_msg>Internal. Reduce code for completion with lua.Lua.PushReflect<commit_after>package mains

import (
	"errors"
	"fmt"
	"strings"

	"../completion"
	"../lua"
	"../readline"
)

var completionHook lua.Pushable = lua.TNil{}

func luaHookForComplete(this *readline.Buffer, rv *completion.List) (*completion.List, error) {
	L, L_ok := this.Context.Value("lua").(lua.Lua)
	if !L_ok {
		return rv, errors.New("listUpComplete: could not get lua instance")
	}

	L.Push(completionHook)
	if L.IsFunction(-1) {
		L.Push(map[string]interface{}{
			"rawword": rv.RawWord,
			"pos":     rv.Pos + 1,
			"text":    rv.AllLine,
			"word":    rv.Word,
		})
		L.NewTable()
		for key, val := range rv.List {
			L.Push(1 + key)
			L.PushString(val.InsertStr)
			L.SetTable(-3)
		}
		L.SetField(-2, "list")
		if err := L.Call(1, 1); err != nil {
			fmt.Println(err)
		}
		if L.IsTable(-1) {
			list := make([]completion.Element, 0, len(rv.List)+32)
			wordUpr := strings.ToUpper(rv.Word)
			for i := 1; true; i++ {
				L.Push(i)
				L.GetTable(-2)
				str, strErr := L.ToString(-1)
				L.Pop(1)
				if strErr != nil || str == "" {
					break
				}
				strUpr := strings.ToUpper(str)
				if strings.HasPrefix(strUpr, wordUpr) {
					list = append(list, completion.Element{str, str})
				}
			}
			if len(list) > 0 {
				rv.List = list
			}
		}
	}
	L.Pop(1) // remove something not function or result-table
	return rv, nil
}
<|endoftext|>


<commit_before>package crawler

import (
	"errors"
	"net/http"
	"time"

	log "github.com/meifamily/logrus"

	"github.com/meifamily/ptt-alertor/models/ptt/article"

	"regexp"

	"strings"

	"strconv"

	"golang.org/x/net/html"
)

const pttHostURL = "https://www.ptt.cc"

func CurrentPage(board string) (int, error) {
	url := makeBoardURL(board, -1)
	rsp, err := fetchHTML(url)
	if err != nil {
		return 0, err
	}
	htmlNodes := parseHTML(rsp)
	paging := traverseHTMLNode(htmlNodes, findPagingBlock)
	for _, page := range paging {
		anchors := traverseHTMLNode(page, findAnchor)
		for _, a := range anchors {
			if strings.Contains(a.FirstChild.Data, "上頁") {
				link := getAnchorLink(a)
				re := regexp.MustCompile("\\d+")
				page, err := strconv.Atoi(re.FindString(link))
				if err != nil {
					return 0, err
				}
				return page + 1, nil
			}
		}
	}
	return 0, errors.New("Parse Currenect Page Error")
}

// BuildArticles makes board's index articles to a article slice
func BuildArticles(board string, page int) (article.Articles, error) {
	reqURL := makeBoardURL(board, page)
	rsp, err := fetchHTML(reqURL)
	if err != nil {
		return article.Articles{}, err
	}
	htmlNodes := parseHTML(rsp)

	articleBlocks := traverseHTMLNode(htmlNodes, findArticleBlocks)
	initialTargetNodes()
	articles := make(article.Articles, len(articleBlocks))
	for index, articleBlock := range articleBlocks {
		for _, pushCountDiv := range traverseHTMLNode(articleBlock, findPushCountDiv) {
			initialTargetNodes()
			if child := pushCountDiv.FirstChild; child != nil {
				if child := child.FirstChild; child != nil {
					if err == nil {
						articles[index].PushCount = convertPushCount(child.Data)
					}
				}
			}
		}
		for _, titleDiv := range traverseHTMLNode(articleBlock, findTitleDiv) {
			initialTargetNodes()

			anchors := traverseHTMLNode(titleDiv, findAnchor)

			if len(anchors) == 0 {
				articles[index].Title = titleDiv.FirstChild.Data
				articles[index].Link = ""
				continue
			}

			for _, anchor := range traverseHTMLNode(titleDiv, findAnchor) {
				articles[index].Title = anchor.FirstChild.Data
				link := pttHostURL + getAnchorLink(anchor)
				articles[index].Link = link
				articles[index].ID = articles[index].ParseID(link)
			}
		}
		for _, metaDiv := range traverseHTMLNode(articleBlock, findMetaDiv) {
			initialTargetNodes()

			for _, date := range traverseHTMLNode(metaDiv, findDateDiv) {
				articles[index].Date = strings.TrimSpace(date.FirstChild.Data)
			}
			for _, author := range traverseHTMLNode(metaDiv, findAuthorDiv) {
				articles[index].Author = author.FirstChild.Data
			}
		}
	}
	return articles, nil
}

func convertPushCount(str string) int {
	switch str {
	case "爆":
		return 100
	case "X1":
		return -10
	case "X2":
		return -20
	case "X3":
		return -30
	case "X4":
		return -40
	case "X5":
		return -50
	case "X6":
		return -60
	case "X7":
		return -70
	case "X8":
		return -80
	case "X9":
		return -90
	case "XX":
		return -100
	default:
		cnt, err := strconv.Atoi(str)
		if err != nil {
			cnt = 0
		}
		return cnt
	}
}

// BuildArticle build article object from html
func BuildArticle(board, articleCode string) (article.Article, error) {

	reqURL := makeArticleURL(board, articleCode)
	rsp, err := fetchHTML(reqURL)
	if err != nil {
		return article.Article{}, err
	}
	htmlNodes := parseHTML(rsp)
	atcl := article.Article{
		Link:  reqURL,
		Code:  articleCode,
		Board: board,
	}
	nodes := traverseHTMLNode(htmlNodes, findOgTitleMeta)
	if len(nodes) > 0 {
		atcl.Title = getMetaContent(nodes[0])
	} else {
		atcl.Title = "[內文標題已被刪除]"
	}
	atcl.ID = atcl.ParseID(reqURL)
	pushBlocks := traverseHTMLNode(htmlNodes, findPushBlocks)
	initialTargetNodes()
	pushes := make([]article.Push, len(pushBlocks))
	for index, pushBlock := range pushBlocks {
		for _, pushTag := range traverseHTMLNode(pushBlock, findPushTag) {
			initialTargetNodes()
			pushes[index].Tag = pushTag.FirstChild.Data
		}
		for _, pushUserID := range traverseHTMLNode(pushBlock, findPushUserID) {
			initialTargetNodes()
			pushes[index].UserID = pushUserID.FirstChild.Data
		}
		for _, pushContent := range traverseHTMLNode(pushBlock, findPushContent) {
			initialTargetNodes()
			content := pushContent.FirstChild.Data
			for n := pushContent.FirstChild.NextSibling; n != nil; n = n.NextSibling {
				if findEmailProtected(n) != nil {
					break
				}
				if n.FirstChild != nil {
					content += n.FirstChild.Data
				}
				if n.NextSibling != nil {
					content += n.NextSibling.Data
				}
			}
			pushes[index].Content = content
		}
		for _, pushIPDateTime := range traverseHTMLNode(pushBlock, findPushIPDateTime) {
			initialTargetNodes()
			ipdatetime := strings.TrimSpace(pushIPDateTime.FirstChild.Data)
			if ipdatetime == "" {
				break
			}
			dateTime, err := parseDateTime(ipdatetime)
			if err != nil {
				log.WithFields(log.Fields{
					"ipdatetime": ipdatetime,
					"board":      board,
					"code":       articleCode,
				}).WithError(err).Error("Parse DateTime Error")
			}
			pushes[index].DateTime = dateTime
			if index == len(pushBlocks)-1 {
				atcl.LastPushDateTime = pushes[index].DateTime
			}
		}
	}
	atcl.PushList = pushes
	return atcl, nil
}

func parseDateTime(ipdatetime string) (time.Time, error) {
	re, _ := regexp.Compile("(\\d+\\.\\d+\\.\\d+\\.\\d+)?\\s*(.*)")
	subMatches := re.FindStringSubmatch(ipdatetime)
	dateTime := strings.TrimSpace(subMatches[len(subMatches)-1])
	loc, err := time.LoadLocation("UTC")
	if err != nil {
		return time.Time{}, err
	}
	t, err := time.ParseInLocation("01/02 15:04", dateTime, loc)
	if err != nil {
		return time.Time{}, err
	}
	t = t.AddDate(getYear(t), 0, 0)
	return t, nil
}

func getYear(pushTime time.Time) int {
	t := time.Now()
	if t.Month() == 1 && pushTime.Month() == 12 {
		return t.Year() - 1
	}
	return t.Year()
}

// CheckBoardExist use for checking board exist or not
func CheckBoardExist(board string) bool {
	reqURL := makeBoardURL(board, -1)
	_, err := fetchHTML(reqURL)
	if _, ok := err.(URLNotFoundError); ok {
		return false
	}
	return true
}

// CheckArticleExist user for checking article exist or not
func CheckArticleExist(board, articleCode string) bool {
	reqURL := makeArticleURL(board, articleCode)
	_, err := fetchHTML(reqURL)
	if _, ok := err.(URLNotFoundError); ok {
		return false
	}
	return true
}

func makeBoardURL(board string, page int) string {
	var pageStr string
	if page < 0 {
		pageStr = ""
	} else {
		pageStr = strconv.Itoa(page)
	}
	return pttHostURL + "/bbs/" + board + "/index" + pageStr + ".html"
}

func makeArticleURL(board, articleCode string) string {
	return pttHostURL + "/bbs/" + board + "/" + articleCode + ".html"
}

type URLNotFoundError struct {
	URL string
}

func (u URLNotFoundError) Error() string {
	return "Fetched URL Not Found"
}

func fetchHTML(reqURL string) (response *http.Response, err error) {

	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return errors.New("Redirect")
		},
	}

	response, err = client.Get(reqURL)

	if response.StatusCode == http.StatusNotFound {
		err = URLNotFoundError{reqURL}
	}

	if err != nil && response.StatusCode == http.StatusFound {
		req := passR18(reqURL)
		response, err = client.Do(req)
	}

	if err != nil {
		log.WithField("url", reqURL).WithError(err).Error("Fetch URL Failed")
	}

	return response, err
}

func passR18(reqURL string) (req *http.Request) {

	req, _ = http.NewRequest("GET", reqURL, nil)

	over18Cookie := http.Cookie{
		Name:       "over18",
		Value:      "1",
		Domain:     "www.ptt.cc",
		Path:       "/",
		RawExpires: "Session",
		MaxAge:     0,
		HttpOnly:   false,
	}

	req.AddCookie(&over18Cookie)

	return req
}

func parseHTML(response *http.Response) *html.Node {
	doc, err := html.Parse(response.Body)
	if err != nil {
		log.Error(err)
	}
	return doc
}
<commit_msg>add currectPage, make buildArticles could specific page, and detect last user article block<commit_after>package crawler

import (
	"errors"
	"net/http"
	"time"

	log "github.com/meifamily/logrus"

	"github.com/meifamily/ptt-alertor/models/ptt/article"

	"regexp"

	"strings"

	"strconv"

	"github.com/meifamily/ptt-alertor/models/pushsum"
	"golang.org/x/net/html"
)

const pttHostURL = "https://www.ptt.cc"

func CurrentPage(board string) (int, error) {
	url := makeBoardURL(board, -1)
	rsp, err := fetchHTML(url)
	if err != nil {
		return 0, err
	}
	htmlNodes := parseHTML(rsp)
	paging := traverseHTMLNode(htmlNodes, findPagingBlock)
	for _, page := range paging {
		anchors := traverseHTMLNode(page, findAnchor)
		for _, a := range anchors {
			if strings.Contains(a.FirstChild.Data, "上頁") {
				link := getAnchorLink(a)
				re := regexp.MustCompile("\\d+")
				page, err := strconv.Atoi(re.FindString(link))
				if err != nil {
					return 0, err
				}
				return page + 1, nil
			}
		}
	}
	return 0, errors.New("Parse Currenect Page Error")
}

// BuildArticles makes board's index articles to a article slice
func BuildArticles(board string, page int) (article.Articles, error) {
	reqURL := makeBoardURL(board, page)
	rsp, err := fetchHTML(reqURL)
	if err != nil {
		return article.Articles{}, err
	}
	htmlNodes := parseHTML(rsp)

	articleBlocks := traverseHTMLNode(htmlNodes, findArticleBlocks)
	initialTargetNodes()
	articles := make(article.Articles, len(articleBlocks))
	for index, articleBlock := range articleBlocks {
		if isLastArticleBlock(articleBlock) {
			break
		}
		for _, pushCountDiv := range traverseHTMLNode(articleBlock, findPushCountDiv) {
			initialTargetNodes()
			if child := pushCountDiv.FirstChild; child != nil {
				if child := child.FirstChild; child != nil {
					if err == nil {
						articles[index].PushSum = convertPushCount(child.Data)
					}
				}
			}
		}
		for _, titleDiv := range traverseHTMLNode(articleBlock, findTitleDiv) {
			initialTargetNodes()

			anchors := traverseHTMLNode(titleDiv, findAnchor)

			if len(anchors) == 0 {
				articles[index].Title = titleDiv.FirstChild.Data
				articles[index].Link = ""
				continue
			}

			for _, anchor := range traverseHTMLNode(titleDiv, findAnchor) {
				articles[index].Title = anchor.FirstChild.Data
				link := pttHostURL + getAnchorLink(anchor)
				articles[index].Link = link
				articles[index].ID = articles[index].ParseID(link)
			}
		}
		for _, metaDiv := range traverseHTMLNode(articleBlock, findMetaDiv) {
			initialTargetNodes()

			for _, date := range traverseHTMLNode(metaDiv, findDateDiv) {
				articles[index].Date = strings.TrimSpace(date.FirstChild.Data)
			}
			for _, author := range traverseHTMLNode(metaDiv, findAuthorDiv) {
				articles[index].Author = author.FirstChild.Data
			}
		}
	}
	return articles, nil
}

func isLastArticleBlock(articleBlock *html.Node) bool {
	for next := articleBlock.NextSibling; ; next = next.NextSibling {
		if next == nil {
			break
		}
		if next.Type == html.ElementNode {
			for _, attr := range next.Attr {
				if attr.Val == "r-list-sep" {
					return true
				}
				return false
			}
		}
	}
	return false
}

func convertPushCount(str string) int {
	for num, text := range pushsum.NumTextMap {
		if strings.EqualFold(str, text) {
			return num
		}
	}
	cnt, err := strconv.Atoi(str)
	if err != nil {
		cnt = 0
	}
	return cnt
}

// BuildArticle build article object from html
func BuildArticle(board, articleCode string) (article.Article, error) {

	reqURL := makeArticleURL(board, articleCode)
	rsp, err := fetchHTML(reqURL)
	if err != nil {
		return article.Article{}, err
	}
	htmlNodes := parseHTML(rsp)
	atcl := article.Article{
		Link:  reqURL,
		Code:  articleCode,
		Board: board,
	}
	nodes := traverseHTMLNode(htmlNodes, findOgTitleMeta)
	if len(nodes) > 0 {
		atcl.Title = getMetaContent(nodes[0])
	} else {
		atcl.Title = "[內文標題已被刪除]"
	}
	atcl.ID = atcl.ParseID(reqURL)
	pushBlocks := traverseHTMLNode(htmlNodes, findPushBlocks)
	initialTargetNodes()
	pushes := make([]article.Push, len(pushBlocks))
	for index, pushBlock := range pushBlocks {
		for _, pushTag := range traverseHTMLNode(pushBlock, findPushTag) {
			initialTargetNodes()
			pushes[index].Tag = pushTag.FirstChild.Data
		}
		for _, pushUserID := range traverseHTMLNode(pushBlock, findPushUserID) {
			initialTargetNodes()
			pushes[index].UserID = pushUserID.FirstChild.Data
		}
		for _, pushContent := range traverseHTMLNode(pushBlock, findPushContent) {
			initialTargetNodes()
			content := pushContent.FirstChild.Data
			for n := pushContent.FirstChild.NextSibling; n != nil; n = n.NextSibling {
				if findEmailProtected(n) != nil {
					break
				}
				if n.FirstChild != nil {
					content += n.FirstChild.Data
				}
				if n.NextSibling != nil {
					content += n.NextSibling.Data
				}
			}
			pushes[index].Content = content
		}
		for _, pushIPDateTime := range traverseHTMLNode(pushBlock, findPushIPDateTime) {
			initialTargetNodes()
			ipdatetime := strings.TrimSpace(pushIPDateTime.FirstChild.Data)
			if ipdatetime == "" {
				break
			}
			dateTime, err := parseDateTime(ipdatetime)
			if err != nil {
				log.WithFields(log.Fields{
					"ipdatetime": ipdatetime,
					"board":      board,
					"code":       articleCode,
				}).WithError(err).Error("Parse DateTime Error")
			}
			pushes[index].DateTime = dateTime
			if index == len(pushBlocks)-1 {
				atcl.LastPushDateTime = pushes[index].DateTime
			}
		}
	}
	atcl.PushList = pushes
	return atcl, nil
}

func parseDateTime(ipdatetime string) (time.Time, error) {
	re, _ := regexp.Compile("(\\d+\\.\\d+\\.\\d+\\.\\d+)?\\s*(.*)")
	subMatches := re.FindStringSubmatch(ipdatetime)
	dateTime := strings.TrimSpace(subMatches[len(subMatches)-1])
	loc, err := time.LoadLocation("UTC")
	if err != nil {
		return time.Time{}, err
	}
	t, err := time.ParseInLocation("01/02 15:04", dateTime, loc)
	if err != nil {
		return time.Time{}, err
	}
	t = t.AddDate(getYear(t), 0, 0)
	return t, nil
}

func getYear(pushTime time.Time) int {
	t := time.Now()
	if t.Month() == 1 && pushTime.Month() == 12 {
		return t.Year() - 1
	}
	return t.Year()
}

// CheckBoardExist use for checking board exist or not
func CheckBoardExist(board string) bool {
	reqURL := makeBoardURL(board, -1)
	_, err := fetchHTML(reqURL)
	if _, ok := err.(URLNotFoundError); ok {
		return false
	}
	return true
}

// CheckArticleExist user for checking article exist or not
func CheckArticleExist(board, articleCode string) bool {
	reqURL := makeArticleURL(board, articleCode)
	_, err := fetchHTML(reqURL)
	if _, ok := err.(URLNotFoundError); ok {
		return false
	}
	return true
}

func makeBoardURL(board string, page int) string {
	var pageStr string
	if page < 0 {
		pageStr = ""
	} else {
		pageStr = strconv.Itoa(page)
	}
	return pttHostURL + "/bbs/" + board + "/index" + pageStr + ".html"
}

func makeArticleURL(board, articleCode string) string {
	return pttHostURL + "/bbs/" + board + "/" + articleCode + ".html"
}

type URLNotFoundError struct {
	URL string
}

func (u URLNotFoundError) Error() string {
	return "Fetched URL Not Found"
}

func fetchHTML(reqURL string) (response *http.Response, err error) {

	client := &http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return errors.New("Redirect")
		},
	}

	response, err = client.Get(reqURL)

	if response.StatusCode == http.StatusNotFound {
		err = URLNotFoundError{reqURL}
	}

	if err != nil && response.StatusCode == http.StatusFound {
		req := passR18(reqURL)
		response, err = client.Do(req)
	}

	if err != nil {
		log.WithField("url", reqURL).WithError(err).Error("Fetch URL Failed")
	}

	return response, err
}

func passR18(reqURL string) (req *http.Request) {

	req, _ = http.NewRequest("GET", reqURL, nil)

	over18Cookie := http.Cookie{
		Name:       "over18",
		Value:      "1",
		Domain:     "www.ptt.cc",
		Path:       "/",
		RawExpires: "Session",
		MaxAge:     0,
		HttpOnly:   false,
	}

	req.AddCookie(&over18Cookie)

	return req
}

func parseHTML(response *http.Response) *html.Node {
	doc, err := html.Parse(response.Body)
	if err != nil {
		log.Error(err)
	}
	return doc
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package jdwp implements types to communicate using the the Java Debug Wire Protocol.
package jdwp

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"reflect"
	"sync"
	"time"

	"github.com/google/gapid/core/app/crash"
	"github.com/google/gapid/core/data/binary"
	"github.com/google/gapid/core/data/endian"
	"github.com/google/gapid/core/os/device"
)

var (
	handshake = []byte("JDWP-Handshake")

	defaultIDSizes = IDSizes{
		FieldIDSize:         8,
		MethodIDSize:        8,
		ObjectIDSize:        8,
		ReferenceTypeIDSize: 8,
		FrameIDSize:         8,
	}
)

type eventsID uint64

// Connection represents a JDWP connection.
type Connection struct {
	in           io.Reader
	r            binary.Reader
	w            binary.Writer
	flush        func() error
	idSizes      IDSizes
	nextPacketID packetID
	nextEventsID eventsID
	events       map[eventsID]chan<- Events
	replies      map[packetID]chan<- replyPacket
	sync.Mutex
}

// Open creates a Connection using conn for I/O.
func Open(ctx context.Context, conn io.ReadWriteCloser) (*Connection, error) {
	if err := exchangeHandshakes(conn); err != nil {
		return nil, err
	}

	buf := bufio.NewWriterSize(conn, 1024)
	r := endian.Reader(conn, device.BigEndian)
	w := endian.Writer(buf, device.BigEndian)
	events := map[eventsID]chan<- Events{}
	replies := map[packetID]chan<- replyPacket{}
	c := &Connection{
		in:      conn,
		r:       r,
		w:       w,
		flush:   buf.Flush,
		idSizes: defaultIDSizes,
		events:  events,
		replies: replies,
	}

	crash.Go(func() { c.recv(ctx) })
	var err error
	c.idSizes, err = c.GetIDSizes()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func exchangeHandshakes(conn io.ReadWriter) error {
	if _, err := conn.Write(handshake); err != nil {
		return err
	}
	ok, err := expect(conn, handshake)
	if err != nil {
		return err
	}
	if !ok {
		return fmt.Errorf("Bad handshake")
	}
	return nil
}

// expect reads c.in, expecting the specfified sequence of bytes. If the read
// data doesn't match, then the function returns immediately with false.
func expect(conn io.Reader, expected []byte) (bool, error) {
	got := make([]byte, len(expected))
	for len(expected) > 0 {
		n, err := conn.Read(got)
		if err != nil {
			return false, err
		}
		for i := 0; i < n; i++ {
			if got[i] != expected[i] {
				return false, nil
			}
		}
		got, expected = got[n:], expected[n:]
	}
	return true, nil
}

// get sends the specified command and waits for a reply.
func (c *Connection) get(cmdSet cmdSet, cmd cmdID, req interface{}, out interface{}) error {
	data := bytes.Buffer{}
	if req != nil {
		e := endian.Writer(&data, device.BigEndian)
		if err := c.encode(e, reflect.ValueOf(req)); err != nil {
			return err
		}
	}

	id, replyChan := c.newReplyHandler()
	defer c.deleteReplyHandler(id)

	p := cmdPacket{id: id, cmdSet: cmdSet, cmdID: cmd, data: data.Bytes()}
	if err := p.write(c.w); err != nil {
		return err
	}
	if err := c.flush(); err != nil {
		return err
	}

	select {
	case reply := <-replyChan:
		if reply.err != ErrNone {
			return reply.err
		}
		if out == nil {
			return nil
		}
		r := bytes.NewReader(reply.data)
		d := endian.Reader(r, device.BigEndian)
		if err := c.decode(d, reflect.ValueOf(out)); err != nil {
			return err
		}
		if offset, _ := r.Seek(0, 1); offset != int64(len(reply.data)) {
			panic(fmt.Errorf("Only %d/%d bytes read from reply packet", offset, len(reply.data)))
		}
		return nil
	case <-time.After(time.Second * 30):
		return fmt.Errorf("timeout")
	}
}

func (c *Connection) newReplyHandler() (packetID, <-chan replyPacket) {
	reply := make(chan replyPacket, 1)
	c.Lock()
	id := c.nextPacketID
	c.nextPacketID++
	c.replies[id] = reply
	c.Unlock()
	return id, reply
}

func (c *Connection) deleteReplyHandler(id packetID) {
	c.Lock()
	delete(c.replies, id)
	c.Unlock()
}
<commit_msg>Increased the timeout for jdwp commands.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package jdwp implements types to communicate using the the Java Debug Wire Protocol.
package jdwp

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"reflect"
	"sync"
	"time"

	"github.com/google/gapid/core/app/crash"
	"github.com/google/gapid/core/data/binary"
	"github.com/google/gapid/core/data/endian"
	"github.com/google/gapid/core/os/device"
)

var (
	handshake = []byte("JDWP-Handshake")

	defaultIDSizes = IDSizes{
		FieldIDSize:         8,
		MethodIDSize:        8,
		ObjectIDSize:        8,
		ReferenceTypeIDSize: 8,
		FrameIDSize:         8,
	}
)

type eventsID uint64

// Connection represents a JDWP connection.
type Connection struct {
	in           io.Reader
	r            binary.Reader
	w            binary.Writer
	flush        func() error
	idSizes      IDSizes
	nextPacketID packetID
	nextEventsID eventsID
	events       map[eventsID]chan<- Events
	replies      map[packetID]chan<- replyPacket
	sync.Mutex
}

// Open creates a Connection using conn for I/O.
func Open(ctx context.Context, conn io.ReadWriteCloser) (*Connection, error) {
	if err := exchangeHandshakes(conn); err != nil {
		return nil, err
	}

	buf := bufio.NewWriterSize(conn, 1024)
	r := endian.Reader(conn, device.BigEndian)
	w := endian.Writer(buf, device.BigEndian)
	events := map[eventsID]chan<- Events{}
	replies := map[packetID]chan<- replyPacket{}
	c := &Connection{
		in:      conn,
		r:       r,
		w:       w,
		flush:   buf.Flush,
		idSizes: defaultIDSizes,
		events:  events,
		replies: replies,
	}

	crash.Go(func() { c.recv(ctx) })
	var err error
	c.idSizes, err = c.GetIDSizes()
	if err != nil {
		return nil, err
	}
	return c, nil
}

func exchangeHandshakes(conn io.ReadWriter) error {
	if _, err := conn.Write(handshake); err != nil {
		return err
	}
	ok, err := expect(conn, handshake)
	if err != nil {
		return err
	}
	if !ok {
		return fmt.Errorf("Bad handshake")
	}
	return nil
}

// expect reads c.in, expecting the specfified sequence of bytes. If the read
// data doesn't match, then the function returns immediately with false.
func expect(conn io.Reader, expected []byte) (bool, error) {
	got := make([]byte, len(expected))
	for len(expected) > 0 {
		n, err := conn.Read(got)
		if err != nil {
			return false, err
		}
		for i := 0; i < n; i++ {
			if got[i] != expected[i] {
				return false, nil
			}
		}
		got, expected = got[n:], expected[n:]
	}
	return true, nil
}

// get sends the specified command and waits for a reply.
func (c *Connection) get(cmdSet cmdSet, cmd cmdID, req interface{}, out interface{}) error {
	data := bytes.Buffer{}
	if req != nil {
		e := endian.Writer(&data, device.BigEndian)
		if err := c.encode(e, reflect.ValueOf(req)); err != nil {
			return err
		}
	}

	id, replyChan := c.newReplyHandler()
	defer c.deleteReplyHandler(id)

	p := cmdPacket{id: id, cmdSet: cmdSet, cmdID: cmd, data: data.Bytes()}
	if err := p.write(c.w); err != nil {
		return err
	}
	if err := c.flush(); err != nil {
		return err
	}

	select {
	case reply := <-replyChan:
		if reply.err != ErrNone {
			return reply.err
		}
		if out == nil {
			return nil
		}
		r := bytes.NewReader(reply.data)
		d := endian.Reader(r, device.BigEndian)
		if err := c.decode(d, reflect.ValueOf(out)); err != nil {
			return err
		}
		if offset, _ := r.Seek(0, 1); offset != int64(len(reply.data)) {
			panic(fmt.Errorf("Only %d/%d bytes read from reply packet", offset, len(reply.data)))
		}
		return nil
	case <-time.After(time.Second * 120):
		return fmt.Errorf("timeout")
	}
}

func (c *Connection) newReplyHandler() (packetID, <-chan replyPacket) {
	reply := make(chan replyPacket, 1)
	c.Lock()
	id := c.nextPacketID
	c.nextPacketID++
	c.replies[id] = reply
	c.Unlock()
	return id, reply
}

func (c *Connection) deleteReplyHandler(id packetID) {
	c.Lock()
	delete(c.replies, id)
	c.Unlock()
}
<|endoftext|>


<commit_before>package core_test

import (
	"github.com/APTrust/bagit/core"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"testing"
)

func TestNewKeyValuePair(t *testing.T) {
	item := core.NewKeyValuePair("key", "value")
	assert.Equal(t, "key", item.Key)
	assert.Equal(t, "value", item.Value)
}

func TestNewKeyValueCollection(t *testing.T) {
	items := core.NewKeyValueCollection()
	require.NotNil(t, items)
}

func TestKeyValueCollectionAppend(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")
	assert.Equal(t, 2, len(items.Keys()))
	assert.Equal(t, 3, len(items.Values()))
}

func TestKeyValueCollectionFindByKey(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")

	matches := items.FindByKey("key1")
	require.Equal(t, 1, len(matches))
	assert.Equal(t, "value1", matches[0].Value)

	matches = items.FindByKey("key2")
	require.Equal(t, 2, len(matches))
	assert.Equal(t, "value2", matches[0].Value)
	assert.Equal(t, "value3", matches[1].Value)

	matches = items.FindByKey("does_not_exist")
	require.Empty(t, matches)
}

func TestKeyValueCollectionFindByValue(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key1", "value2")
	items.Append("key2", "value2")
	items.Append("key2", "value3")

	matches := items.FindByValue("value1")
	require.Equal(t, 1, len(matches))
	assert.Equal(t, "key1", matches[0].Key)

	matches = items.FindByValue("value2")
	require.Equal(t, 2, len(matches))
	assert.Equal(t, "key1", matches[0].Key)
	assert.Equal(t, "key2", matches[1].Key)

	matches = items.FindByValue("does_not_exist")
	require.Empty(t, matches)
}

func TestKeyValueCollectionValuesForKey(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")

	matches := items.ValuesForKey("key1")
	require.Equal(t, 1, len(matches))
	assert.Equal(t, "value1", matches[0])

	matches = items.ValuesForKey("key2")
	require.Equal(t, 2, len(matches))
	assert.Equal(t, "value2", matches[0])
	assert.Equal(t, "value3", matches[1])

	matches = items.ValuesForKey("does_not_exist")
	require.Empty(t, matches)
}

func TestKeyValueCollectionFirstValueForKey(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key1", "value2")
	items.Append("key1", "value3")
	items.Append("key2", "value10")
	items.Append("key2", "value20")

	assert.Equal(t, "value1", items.FirstValueForKey("key1"))
	assert.Equal(t, "value10", items.FirstValueForKey("key2"))

	require.Empty(t, items.FirstValueForKey("no_such_key"))
}

func TestKeyValueCollectionCount(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")
	assert.Equal(t, 3, items.Count())

	items.Append("key3", "value4")
	assert.Equal(t, 4, items.Count())
}

func TestKeyValueCollectionDelete(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	item_2_2 := items.Append("key2", "value2")
	items.Append("key2", "value3")
	assert.Equal(t, 3, items.Count())

	items.Delete(item_2_2)
	assert.Equal(t, 2, items.Count())
	matches := items.FindByKey("key2")
	for _, match := range matches {
		assert.NotEqual(t, "value2", match.Value)
	}
}

func TestKeyValueCollectionDeleteByKey(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")
	assert.Equal(t, 3, items.Count())

	items.DeleteByKey("key2")
	assert.Equal(t, 1, items.Count())
	assert.Empty(t, items.FindByKey("key2"))
}
<commit_msg>Added test for Items<commit_after>package core_test

import (
	"github.com/APTrust/bagit/core"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"testing"
)

func TestNewKeyValuePair(t *testing.T) {
	item := core.NewKeyValuePair("key", "value")
	assert.Equal(t, "key", item.Key)
	assert.Equal(t, "value", item.Value)
}

func TestNewKeyValueCollection(t *testing.T) {
	items := core.NewKeyValueCollection()
	require.NotNil(t, items)
}

func TestKeyValueCollectionAppend(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")
	assert.Equal(t, 2, len(items.Keys()))
	assert.Equal(t, 3, len(items.Values()))
}

func TestKeyValueCollectionFindByKey(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")

	matches := items.FindByKey("key1")
	require.Equal(t, 1, len(matches))
	assert.Equal(t, "value1", matches[0].Value)

	matches = items.FindByKey("key2")
	require.Equal(t, 2, len(matches))
	assert.Equal(t, "value2", matches[0].Value)
	assert.Equal(t, "value3", matches[1].Value)

	matches = items.FindByKey("does_not_exist")
	require.Empty(t, matches)
}

func TestKeyValueCollectionFindByValue(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key1", "value2")
	items.Append("key2", "value2")
	items.Append("key2", "value3")

	matches := items.FindByValue("value1")
	require.Equal(t, 1, len(matches))
	assert.Equal(t, "key1", matches[0].Key)

	matches = items.FindByValue("value2")
	require.Equal(t, 2, len(matches))
	assert.Equal(t, "key1", matches[0].Key)
	assert.Equal(t, "key2", matches[1].Key)

	matches = items.FindByValue("does_not_exist")
	require.Empty(t, matches)
}

func TestKeyValueCollectionValuesForKey(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")

	matches := items.ValuesForKey("key1")
	require.Equal(t, 1, len(matches))
	assert.Equal(t, "value1", matches[0])

	matches = items.ValuesForKey("key2")
	require.Equal(t, 2, len(matches))
	assert.Equal(t, "value2", matches[0])
	assert.Equal(t, "value3", matches[1])

	matches = items.ValuesForKey("does_not_exist")
	require.Empty(t, matches)
}

func TestKeyValueCollectionFirstValueForKey(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key1", "value2")
	items.Append("key1", "value3")
	items.Append("key2", "value10")
	items.Append("key2", "value20")

	assert.Equal(t, "value1", items.FirstValueForKey("key1"))
	assert.Equal(t, "value10", items.FirstValueForKey("key2"))

	require.Empty(t, items.FirstValueForKey("no_such_key"))
}

func TestKeyValueCollectionItems(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")

	kvPairs := items.Items()
	assert.Equal(t, 3, len(kvPairs))
	assert.Equal(t, "key2", kvPairs[1].Key)
	assert.Equal(t, "value2", kvPairs[1].Value)
}

func TestKeyValueCollectionCount(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")
	assert.Equal(t, 3, items.Count())

	items.Append("key3", "value4")
	assert.Equal(t, 4, items.Count())
}

func TestKeyValueCollectionDelete(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	item_2_2 := items.Append("key2", "value2")
	items.Append("key2", "value3")
	assert.Equal(t, 3, items.Count())

	items.Delete(item_2_2)
	assert.Equal(t, 2, items.Count())
	matches := items.FindByKey("key2")
	for _, match := range matches {
		assert.NotEqual(t, "value2", match.Value)
	}
}

func TestKeyValueCollectionDeleteByKey(t *testing.T) {
	items := core.NewKeyValueCollection()
	items.Append("key1", "value1")
	items.Append("key2", "value2")
	items.Append("key2", "value3")
	assert.Equal(t, 3, items.Count())

	items.DeleteByKey("key2")
	assert.Equal(t, 1, items.Count())
	assert.Empty(t, items.FindByKey("key2"))
}
<|endoftext|>


<commit_before>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/blang/semver"
	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"net/url"
)

// ListItem describes Gorjun entity. It can be APT package, Subutai template or Raw file.
type ListItem struct {
	ID           string            `json:"id"`
	Hash         hashsums          `json:"hash"`
	Size         int               `json:"size"`
	Date         time.Time         `json:"upload-date-formatted"`
	Timestamp    string            `json:"upload-date-timestamp,omitempty"`
	Name         string            `json:"name,omitempty"`
	Tags         []string          `json:"tags,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type hashsums struct {
	Md5    string `json:"md5,omitempty"`
	Sha256 string `json:"sha256,omitempty"`
}

// Handler provides download functionality for all artifacts.
func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	if len(id) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify id or name")
		return
	} else if len(name) != 0 {
		id = db.LastHash(name, repo)
	}

	if len(db.Read(id)) > 0 && !db.Public(id) && !db.CheckShare(id, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	path := config.Storage.Path + id
	if md5, _ := db.Hash(id); len(md5) != 0 {
		path = config.Storage.Path + md5
	}

	f, err := os.Open(path)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+id, err) || len(id) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(id); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + id)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(id)+"\"")
	}

	io.Copy(w, f)
}

// Info returns JSON formatted list of elements. It allows to apply some filters to Search.
func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var fullname bool
	var itemLatestVersion ListItem
	p := []int{0, 1000}
	id := r.URL.Query().Get("id")
	tag := r.URL.Query().Get("tag")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	subname := r.URL.Query().Get("subname")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")
	if len(subname) != 0 {
		name = subname
	}

	list := db.Search(name)
	if len(tag) > 0 {
		listByTag, err := db.Tag(tag)
		log.Check(log.DebugLevel, "Looking for artifacts with tag "+tag, err)
		list = intersect(list, listByTag)
	}
	if onlyOneParameterProvided("name", r) {
		verified = "true"
	}
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items = append(items, getVerified(list, name, repo))
		items[0].Signature = db.FileSignatures(items[0].ID)
		output, err := json.Marshal(items)
		if err == nil && len(items) > 0 && items[0].ID != "" {
			return output
		}
		return nil
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}
	latestVersion, _ := semver.Make("")
	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if p[0]--; p[0] > 0 {
			continue
		}

		item := FormatItem(db.Info(k), repo, name)
		if len(subname) == 0 && name == item.Name {
			if strings.HasSuffix(item.Version, version) || len(version) == 0 {
				items = []ListItem{item}
				fullname = true
				itemVersion, _ := semver.Make(item.Version)
				if itemVersion.GTE(latestVersion) {
					latestVersion = itemVersion
					itemLatestVersion = item
				}
			}
		} else if !fullname && (len(version) == 0 || item.Version == version) {
			items = append(items, item)
		}

		if len(items) >= p[1] {
			break
		}
	}
	if len(items) == 1 {
		if version == "" && repo == "template" && itemLatestVersion.ID != "" {
			items[0] = itemLatestVersion
		}
		items[0].Signature = db.FileSignatures(items[0].ID)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	latestVersion, _ := semver.Make("")
	var itemLatestVersion ListItem
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileField(info["id"], "owner") {
					itemVersion, _ := semver.Make(info["version"])
					if in(owner, []string{"subutai", "jenkins", "docker"}) &&
						itemVersion.GTE(latestVersion) {
						latestVersion = itemVersion
						itemLatestVersion = FormatItem(db.Info(k), repo, name)
					}
				}
			}
		}
	}
	return itemLatestVersion
}

func FormatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}

	date, _ := time.Parse(time.RFC3339Nano, info["date"])
	timestamp := strconv.FormatInt(date.Unix(), 10)
	item := ListItem{
		ID:           info["id"],
		Date:         date,
		Hash:         hashsums{Md5: info["md5"], Sha256: info["sha256"]},
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Tags:         db.FileField(info["id"], "tags"),
		Owner:        db.FileField(info["id"], "owner"),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Description:  info["Description"],
		Timestamp:    timestamp,
	}
	item.Size, _ = strconv.Atoi(info["size"])

	if repo == "apt" {
		item.Version = info["Version"]
		item.Architecture = info["Architecture"]
		item.Size, _ = strconv.Atoi(info["Size"])
	}
	if len(item.Hash.Md5) == 0 {
		item.Hash.Md5 = item.ID
	}
	return item
}

func intersect(listA, listB []string) (list []string) {
	mapA := map[string]bool{}
	for _, item := range listA {
		mapA[item] = true
	}
	for _, item := range listB {
		if mapA[item] {
			list = append(list, item)
		}
	}
	return list
}

func onlyOneParameterProvided(parameter string, r *http.Request) bool {
	u, _ := url.Parse(r.RequestURI)
	parameters, _ := url.ParseQuery(u.RawQuery)
	//if len(parameters["token"]) > 0 && parameters["token"][0] == "" {
	//	return true
	//}
	for key, _ := range parameters {
		if key != parameter {
			return false
		}
	}
	return len(parameters) > 0
}
<commit_msg>Hot-fix for empty token<commit_after>package download

import (
	"crypto/tls"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/blang/semver"
	"github.com/subutai-io/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"net/url"
)

// ListItem describes Gorjun entity. It can be APT package, Subutai template or Raw file.
type ListItem struct {
	ID           string            `json:"id"`
	Hash         hashsums          `json:"hash"`
	Size         int               `json:"size"`
	Date         time.Time         `json:"upload-date-formatted"`
	Timestamp    string            `json:"upload-date-timestamp,omitempty"`
	Name         string            `json:"name,omitempty"`
	Tags         []string          `json:"tags,omitempty"`
	Owner        []string          `json:"owner,omitempty"`
	Parent       string            `json:"parent,omitempty"`
	Version      string            `json:"version,omitempty"`
	Filename     string            `json:"filename,omitempty"`
	Prefsize     string            `json:"prefsize,omitempty"`
	Signature    map[string]string `json:"signature,omitempty"`
	Description  string            `json:"description,omitempty"`
	Architecture string            `json:"architecture,omitempty"`
}

type hashsums struct {
	Md5    string `json:"md5,omitempty"`
	Sha256 string `json:"sha256,omitempty"`
}

// Handler provides download functionality for all artifacts.
func Handler(repo string, w http.ResponseWriter, r *http.Request) {
	id := r.URL.Query().Get("id")
	name := r.URL.Query().Get("name")
	if len(id) == 0 && len(name) == 0 {
		io.WriteString(w, "Please specify id or name")
		return
	} else if len(name) != 0 {
		id = db.LastHash(name, repo)
	}

	if len(db.Read(id)) > 0 && !db.Public(id) && !db.CheckShare(id, db.CheckToken(r.URL.Query().Get("token"))) {
		w.WriteHeader(http.StatusNotFound)
		w.Write([]byte("Not found"))
		return
	}

	path := config.Storage.Path + id
	if md5, _ := db.Hash(id); len(md5) != 0 {
		path = config.Storage.Path + md5
	}

	f, err := os.Open(path)
	defer f.Close()

	if log.Check(log.WarnLevel, "Opening file "+config.Storage.Path+id, err) || len(id) == 0 {
		if len(config.CDN.Node) > 0 {
			client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
			resp, err := client.Get(config.CDN.Node + r.URL.RequestURI())
			if !log.Check(log.WarnLevel, "Getting file from CDN", err) {
				w.Header().Set("Content-Length", resp.Header.Get("Content-Length"))
				w.Header().Set("Content-Type", resp.Header.Get("Content-Type"))
				w.Header().Set("Last-Modified", resp.Header.Get("Last-Modified"))
				w.Header().Set("Content-Disposition", resp.Header.Get("Content-Disposition"))

				io.Copy(w, resp.Body)
				resp.Body.Close()
				return
			}
		}

		w.WriteHeader(http.StatusNotFound)
		io.WriteString(w, "File not found")
		return
	}
	fi, _ := f.Stat()

	if t, err := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && fi.ModTime().Unix() <= t.Unix() {
		w.WriteHeader(http.StatusNotModified)
		return
	}

	w.Header().Set("Content-Length", fmt.Sprint(fi.Size()))
	w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
	w.Header().Set("Last-Modified", fi.ModTime().Format(http.TimeFormat))

	if name = db.Read(id); len(name) == 0 && len(config.CDN.Node) > 0 {
		httpclient := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
		resp, err := httpclient.Get(config.CDN.Node + "/kurjun/rest/template/info?id=" + id)
		if !log.Check(log.WarnLevel, "Getting info from CDN", err) {
			var info ListItem
			rsp, err := ioutil.ReadAll(resp.Body)
			if log.Check(log.WarnLevel, "Reading from CDN response", err) {
				w.WriteHeader(http.StatusNotFound)
				io.WriteString(w, "File not found")
				return
			}
			if !log.Check(log.WarnLevel, "Decrypting request", json.Unmarshal([]byte(rsp), &info)) {
				w.Header().Set("Content-Disposition", "attachment; filename=\""+info.Filename+"\"")
			}
			resp.Body.Close()
		}
	} else {
		w.Header().Set("Content-Disposition", "attachment; filename=\""+db.Read(id)+"\"")
	}

	io.Copy(w, f)
}

// Info returns JSON formatted list of elements. It allows to apply some filters to Search.
func Info(repo string, r *http.Request) []byte {
	var items []ListItem
	var fullname bool
	var itemLatestVersion ListItem
	p := []int{0, 1000}
	id := r.URL.Query().Get("id")
	tag := r.URL.Query().Get("tag")
	name := r.URL.Query().Get("name")
	page := r.URL.Query().Get("page")
	owner := r.URL.Query().Get("owner")
	token := r.URL.Query().Get("token")
	subname := r.URL.Query().Get("subname")
	version := r.URL.Query().Get("version")
	verified := r.URL.Query().Get("verified")
	if len(subname) != 0 {
		name = subname
	}

	list := db.Search(name)
	if len(tag) > 0 {
		listByTag, err := db.Tag(tag)
		log.Check(log.DebugLevel, "Looking for artifacts with tag "+tag, err)
		list = intersect(list, listByTag)
	}
	if onlyOneParameterProvided("name", r) {
		verified = "true"
	}
	if len(id) > 0 {
		list = append(list[:0], id)
	} else if verified == "true" {
		items = append(items, getVerified(list, name, repo))
		items[0].Signature = db.FileSignatures(items[0].ID)
		output, err := json.Marshal(items)
		if err == nil && len(items) > 0 && items[0].ID != "" {
			return output
		}
		return nil
	}

	pstr := strings.Split(page, ",")
	p[0], _ = strconv.Atoi(pstr[0])
	if len(pstr) == 2 {
		p[1], _ = strconv.Atoi(pstr[1])
	}
	latestVersion, _ := semver.Make("")
	for _, k := range list {
		if (!db.Public(k) && !db.CheckShare(k, db.CheckToken(token))) ||
			(len(owner) > 0 && db.CheckRepo(owner, repo, k) == 0) ||
			db.CheckRepo("", repo, k) == 0 {
			continue
		}

		if p[0]--; p[0] > 0 {
			continue
		}

		item := FormatItem(db.Info(k), repo, name)
		if len(subname) == 0 && name == item.Name {
			if strings.HasSuffix(item.Version, version) || len(version) == 0 {
				items = []ListItem{item}
				fullname = true
				itemVersion, _ := semver.Make(item.Version)
				if itemVersion.GTE(latestVersion) {
					latestVersion = itemVersion
					itemLatestVersion = item
				}
			}
		} else if !fullname && (len(version) == 0 || item.Version == version) {
			items = append(items, item)
		}

		if len(items) >= p[1] {
			break
		}
	}
	if len(items) == 1 {
		if version == "" && repo == "template" && itemLatestVersion.ID != "" {
			items[0] = itemLatestVersion
		}
		items[0].Signature = db.FileSignatures(items[0].ID)
	}
	output, err := json.Marshal(items)
	if err != nil || string(output) == "null" {
		return nil
	}
	return output
}

func in(str string, list []string) bool {
	for _, s := range list {
		if s == str {
			return true
		}
	}
	return false
}

func getVerified(list []string, name, repo string) ListItem {
	latestVersion, _ := semver.Make("")
	var itemLatestVersion ListItem
	for _, k := range list {
		if info := db.Info(k); db.CheckRepo("", repo, k) > 0 {
			if info["name"] == name || (strings.HasPrefix(info["name"], name+"-subutai-template") && repo == "template") {
				for _, owner := range db.FileField(info["id"], "owner") {
					itemVersion, _ := semver.Make(info["version"])
					if in(owner, []string{"subutai", "jenkins", "docker"}) &&
						itemVersion.GTE(latestVersion) {
						latestVersion = itemVersion
						itemLatestVersion = FormatItem(db.Info(k), repo, name)
					}
				}
			}
		}
	}
	return itemLatestVersion
}

func FormatItem(info map[string]string, repo, name string) ListItem {
	if len(info["prefsize"]) == 0 && repo == "template" {
		info["prefsize"] = "tiny"
	}

	date, _ := time.Parse(time.RFC3339Nano, info["date"])
	timestamp := strconv.FormatInt(date.Unix(), 10)
	item := ListItem{
		ID:           info["id"],
		Date:         date,
		Hash:         hashsums{Md5: info["md5"], Sha256: info["sha256"]},
		Name:         strings.Split(info["name"], "-subutai-template")[0],
		Tags:         db.FileField(info["id"], "tags"),
		Owner:        db.FileField(info["id"], "owner"),
		Version:      info["version"],
		Filename:     info["name"],
		Parent:       info["parent"],
		Prefsize:     info["prefsize"],
		Architecture: strings.ToUpper(info["arch"]),
		Description:  info["Description"],
		Timestamp:    timestamp,
	}
	item.Size, _ = strconv.Atoi(info["size"])

	if repo == "apt" {
		item.Version = info["Version"]
		item.Architecture = info["Architecture"]
		item.Size, _ = strconv.Atoi(info["Size"])
	}
	if len(item.Hash.Md5) == 0 {
		item.Hash.Md5 = item.ID
	}
	return item
}

func intersect(listA, listB []string) (list []string) {
	mapA := map[string]bool{}
	for _, item := range listA {
		mapA[item] = true
	}
	for _, item := range listB {
		if mapA[item] {
			list = append(list, item)
		}
	}
	return list
}

func onlyOneParameterProvided(parameter string, r *http.Request) bool {
	u, _ := url.Parse(r.RequestURI)
	parameters, _ := url.ParseQuery(u.RawQuery)
	for key, _ := range parameters {
		if key != parameter {
			return false
		}
	}
	return len(parameters) > 0
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/olekukonko/tablewriter"

	"gopkg.in/alecthomas/kingpin.v2"
	"gopkg.in/yaml.v2"
)

type config struct {
	BaseURL  string             `yaml:"baseURL"`
	Formats  map[string]format  `yaml:"formats,flow"`
	Elements map[string]element `yaml:"elements,flow"`
}

type element struct {
	ID     string   `yaml:"id"`
	File   string   `yaml:"file"`
	Meta   bool     `yaml:"meta"`
	Name   string   `yaml:"name"`
	Files  []string `yaml:"files"`
	Parent string   `yaml:"parent"`
}

type format struct {
	ID  string `yaml:"ext"`
	Loc string `yaml:"loc"`
}

var (
	app        = kingpin.New("download-geofabrik", "A command-line tool for downloading OSM files.")
	Fconfig    = app.Flag("config", "Set Config file.").Default("./geofabrik.yml").Short('c').String()
	nodownload = app.Flag("nodownload", "Do not download file (test only)").Short('n').Bool()
	verbose    = app.Flag("verbose", "Be verbose").Short('v').Bool()

	update = app.Command("update", "Update geofabrik.yml from github")
	url    = update.Flag("url", "Url for config source").Default("https://raw.githubusercontent.com/julien-noblet/download-geofabrik/stable/geofabrik.yml").String()

	list = app.Command("list", "Show elements available")

	download = app.Command("download", "Download element") //TODO : add d as command
	delement = download.Arg("element", "OSM element").Required().String()
	dosmBz2  = download.Flag("osm.bz2", "Download osm.bz2 if available").Short('B').Bool()
	dshpZip  = download.Flag("shp.zip", "Download shp.zip if available").Short('S').Bool()
	dosmPbf  = download.Flag("osm.pbf", "Download osm.pbf (default)").Short('P').Bool()
	doshPbf  = download.Flag("osh.pbf", "Download osh.pbf (default)").Short('H').Bool()
	dstate   = download.Flag("state", "Download state.txt file").Short('s').Bool()
	dpoly    = download.Flag("poly", "Download poly file").Short('p').Bool()
)

func (e *element) hasParent() bool {
	return len(e.Parent) != 0
}

func miniFormats(s []string) string {
	res := make([]string, 6)
	for _, item := range s {
		switch item {
		case "state":
			res[0] = "s"
		case "osm.pbf":
			res[1] = "P"
		case "osm.bz2":
			res[2] = "B"
		case "osh.pbf":
			res[3] = "H"
		case "poly":
			res[4] = "p"
		case "shp.zip":
			res[5] = "S"
		}
	}

	return strings.Join(res, "")
}

func downloadFromURL(url string, fileName string) {
	if *verbose == true {
		log.Println(" Downloading", url, "to", fileName)
	}

	if *nodownload == false {
		// TODO: check file existence first with io.IsExist
		output, err := os.Create(fileName)
		if err != nil {
			log.Fatalln(" Error while creating ", fileName, "-", err)
			return
		}
		defer output.Close()

		response, err := http.Get(url)
		if err != nil {
			log.Fatalln(" Error while downloading ", url, "-", err)
			return
		}
		defer response.Body.Close()

		n, err := io.Copy(output, response.Body)
		if err != nil {
			log.Fatalln(" Error while downloading ", url, "-", err)
			return
		}

		if *verbose == true {
			log.Println(" ", n, "bytes downloaded.")
		}
	}
}
func elem2preURL(c config, e element) string {
	var res string
	if e.hasParent() {
		res = elem2preURL(c, findElem(c, e.Parent)) + "/"
		if e.File != "" { //TODO use file in config???
			res = res + e.File
		} else {
			res = res + e.ID
		}
	} else {
		res = c.BaseURL + "/" + e.ID
	}
	return res
}

func elem2URL(c config, e element, ext string) string {
	res := elem2preURL(c, e)
	res += c.Formats[ext].Loc
	if !stringInSlice(ext, e.Files) {
		log.Fatalln(" Error!!! " + res + " not exist")
	}

	return res
}

func findElem(c config, e string) element {
	res := c.Elements[e]
	if res.ID == "" {
		log.Fatalln(" " + e + " is not in config! Please use \"list\" command!")
	}
	return res
}
func getFormats() []string {
	var formatFile []string
	if *dosmPbf {
		formatFile = append(formatFile, "osm.pbf")
	}
	if *doshPbf {
		formatFile = append(formatFile, "osh.pbf")
	}
	if *dosmBz2 {
		formatFile = append(formatFile, "osm.bz2")
	}
	if *dshpZip {
		formatFile = append(formatFile, "shp.zip")
	}
	if *dstate {
		formatFile = append(formatFile, "state")
	}
	if *dpoly {
		formatFile = append(formatFile, "poly")
	}
	if len(formatFile) == 0 {
		formatFile = append(formatFile, "osm.pbf")
	}
	return formatFile
}

func listAllRegions(c config) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeader([]string{"ShortName", "Is in", "Long Name", "formats"})
	keys := make(sort.StringSlice, len(c.Elements))
	i := 0
	for k := range c.Elements {
		keys[i] = k
		i++
	}
	keys.Sort()
	for _, item := range keys {
		table.Append([]string{item, c.Elements[c.Elements[item].Parent].Name, c.Elements[item].Name, miniFormats(c.Elements[item].Files)})
	}
	table.Render()
	fmt.Printf("Total elements: %#v\n", len(c.Elements))
}

func loadConfig(configFile string) config {
	filename, _ := filepath.Abs(configFile)
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalln(" File error: %v ", err)
		os.Exit(1)
	}
	var myConfig config
	err = yaml.Unmarshal(file, &myConfig)
	if err != nil {
		panic(err)
	}
	return myConfig

}
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func UpdateConfig(url string, myconfig string) {
	downloadFromURL(url, myconfig)
	fmt.Println("Congratulation, you have the latest geofabrik.yml")
}

func main() {
	switch kingpin.MustParse(app.Parse(os.Args[1:])) {

	case list.FullCommand():
		listAllRegions(loadConfig(*Fconfig))
	case update.FullCommand():
		UpdateConfig(*url, *Fconfig)
	case download.FullCommand():
		formatFile := getFormats()
		for _, format := range formatFile {
			downloadFromURL(elem2URL(loadConfig(*Fconfig), findElem(loadConfig(*Fconfig), *delement), format), *delement+"."+format)
		}
	}
}
<commit_msg>feat(List): Add Markdown support for listing<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/olekukonko/tablewriter"

	"gopkg.in/alecthomas/kingpin.v2"
	"gopkg.in/yaml.v2"
)

type config struct {
	BaseURL  string             `yaml:"baseURL"`
	Formats  map[string]format  `yaml:"formats,flow"`
	Elements map[string]element `yaml:"elements,flow"`
}

type element struct {
	ID     string   `yaml:"id"`
	File   string   `yaml:"file"`
	Meta   bool     `yaml:"meta"`
	Name   string   `yaml:"name"`
	Files  []string `yaml:"files"`
	Parent string   `yaml:"parent"`
}

type format struct {
	ID  string `yaml:"ext"`
	Loc string `yaml:"loc"`
}

var (
	app        = kingpin.New("download-geofabrik", "A command-line tool for downloading OSM files.")
	Fconfig    = app.Flag("config", "Set Config file.").Default("./geofabrik.yml").Short('c').String()
	nodownload = app.Flag("nodownload", "Do not download file (test only)").Short('n').Bool()
	verbose    = app.Flag("verbose", "Be verbose").Short('v').Bool()

	update = app.Command("update", "Update geofabrik.yml from github")
	url    = update.Flag("url", "Url for config source").Default("https://raw.githubusercontent.com/julien-noblet/download-geofabrik/stable/geofabrik.yml").String()

	list = app.Command("list", "Show elements available")
	lmd  = list.Flag("markdown", "generate list in Markdown format").Bool()

	download = app.Command("download", "Download element") //TODO : add d as command
	delement = download.Arg("element", "OSM element").Required().String()
	dosmBz2  = download.Flag("osm.bz2", "Download osm.bz2 if available").Short('B').Bool()
	dshpZip  = download.Flag("shp.zip", "Download shp.zip if available").Short('S').Bool()
	dosmPbf  = download.Flag("osm.pbf", "Download osm.pbf (default)").Short('P').Bool()
	doshPbf  = download.Flag("osh.pbf", "Download osh.pbf (default)").Short('H').Bool()
	dstate   = download.Flag("state", "Download state.txt file").Short('s').Bool()
	dpoly    = download.Flag("poly", "Download poly file").Short('p').Bool()
)

func (e *element) hasParent() bool {
	return len(e.Parent) != 0
}

func miniFormats(s []string) string {
	res := make([]string, 6)
	for _, item := range s {
		switch item {
		case "state":
			res[0] = "s"
		case "osm.pbf":
			res[1] = "P"
		case "osm.bz2":
			res[2] = "B"
		case "osh.pbf":
			res[3] = "H"
		case "poly":
			res[4] = "p"
		case "shp.zip":
			res[5] = "S"
		}
	}

	return strings.Join(res, "")
}

func downloadFromURL(url string, fileName string) {
	if *verbose == true {
		log.Println(" Downloading", url, "to", fileName)
	}

	if *nodownload == false {
		// TODO: check file existence first with io.IsExist
		output, err := os.Create(fileName)
		if err != nil {
			log.Fatalln(" Error while creating ", fileName, "-", err)
			return
		}
		defer output.Close()

		response, err := http.Get(url)
		if err != nil {
			log.Fatalln(" Error while downloading ", url, "-", err)
			return
		}
		defer response.Body.Close()

		n, err := io.Copy(output, response.Body)
		if err != nil {
			log.Fatalln(" Error while downloading ", url, "-", err)
			return
		}

		if *verbose == true {
			log.Println(" ", n, "bytes downloaded.")
		}
	}
}
func elem2preURL(c config, e element) string {
	var res string
	if e.hasParent() {
		res = elem2preURL(c, findElem(c, e.Parent)) + "/"
		if e.File != "" { //TODO use file in config???
			res = res + e.File
		} else {
			res = res + e.ID
		}
	} else {
		res = c.BaseURL + "/" + e.ID
	}
	return res
}

func elem2URL(c config, e element, ext string) string {
	res := elem2preURL(c, e)
	res += c.Formats[ext].Loc
	if !stringInSlice(ext, e.Files) {
		log.Fatalln(" Error!!! " + res + " not exist")
	}

	return res
}

func findElem(c config, e string) element {
	res := c.Elements[e]
	if res.ID == "" {
		log.Fatalln(" " + e + " is not in config! Please use \"list\" command!")
	}
	return res
}
func getFormats() []string {
	var formatFile []string
	if *dosmPbf {
		formatFile = append(formatFile, "osm.pbf")
	}
	if *doshPbf {
		formatFile = append(formatFile, "osh.pbf")
	}
	if *dosmBz2 {
		formatFile = append(formatFile, "osm.bz2")
	}
	if *dshpZip {
		formatFile = append(formatFile, "shp.zip")
	}
	if *dstate {
		formatFile = append(formatFile, "state")
	}
	if *dpoly {
		formatFile = append(formatFile, "poly")
	}
	if len(formatFile) == 0 {
		formatFile = append(formatFile, "osm.pbf")
	}
	return formatFile
}

func listAllRegions(c config,format string) {
	table := tablewriter.NewWriter(os.Stdout)
	table.SetAlignment(tablewriter.ALIGN_LEFT)
	table.SetHeader([]string{"ShortName", "Is in", "Long Name", "formats"})
	if format == "Markdown"{
		table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
		table.SetCenterSeparator("|")
	}
	keys := make(sort.StringSlice, len(c.Elements))
	i := 0
	for k := range c.Elements {
		keys[i] = k
		i++
	}
	keys.Sort()
	for _, item := range keys {
		table.Append([]string{item, c.Elements[c.Elements[item].Parent].Name, c.Elements[item].Name, miniFormats(c.Elements[item].Files)})
	}
	table.Render()
	fmt.Printf("Total elements: %#v\n", len(c.Elements))
}

func loadConfig(configFile string) config {
	filename, _ := filepath.Abs(configFile)
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		log.Fatalln(" File error: %v ", err)
		os.Exit(1)
	}
	var myConfig config
	err = yaml.Unmarshal(file, &myConfig)
	if err != nil {
		panic(err)
	}
	return myConfig

}
func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func UpdateConfig(url string, myconfig string) {
	downloadFromURL(url, myconfig)
	fmt.Println("Congratulation, you have the latest geofabrik.yml")
}

func main() {
	switch kingpin.MustParse(app.Parse(os.Args[1:])) {

	case list.FullCommand():
		var format = ""
		if *lmd {
			format = "Markdown"
		}
		listAllRegions(loadConfig(*Fconfig),format)
	case update.FullCommand():
		UpdateConfig(*url, *Fconfig)
	case download.FullCommand():
		formatFile := getFormats()
		for _, format := range formatFile {
			downloadFromURL(elem2URL(loadConfig(*Fconfig), findElem(loadConfig(*Fconfig), *delement), format), *delement+"."+format)
		}
	}
}
<|endoftext|>


<commit_before>/*
  Copyright 2012 Sergey Cherepanov (https://github.com/cheggaaa)

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

package cnst

const (
	VERSION = "0.3.0 beta"
	SIGN    = "Anteater " + VERSION
)

<commit_msg>0.3.1<commit_after>/*
  Copyright 2012 Sergey Cherepanov (https://github.com/cheggaaa)

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
*/

package cnst

const (
	VERSION = "0.3.1"
	SIGN    = "Anteater " + VERSION
)

<|endoftext|>