<commit_before>package zim

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
	"syscall"
)

const (
	zimHeader = 72173914
	chunkSize = (1 << 31) - 2
)

// ZimReader keep tracks of everything related to ZIM reading
type ZimReader struct {
	f             *os.File
	mmap          []byte
	ArticleCount  uint32
	clusterCount  uint32
	urlPtrPos     uint64
	titlePtrPos   uint64
	clusterPtrPos uint64
	mimeListPos   uint64
	mainPage      uint32
	layoutPage    uint32
	mimeTypeList  []string
	// this mutex is used on 32 bits system only to securise access to one mmap at a time
	sync.Mutex
	// currentSeg indicates which segment is currently mmaped
	currentSeg int
	// totalSeg indicates how many chunks are used to slice the zim file (file size / chunkSize)
	totalSeg int
}

// create a new zim reader
func NewReader(path string) (*ZimReader, error) {
	f, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	z := ZimReader{f: f, mainPage: 0xffffffff, layoutPage: 0xffffffff}

	fi, err := f.Stat()
	if err != nil {
		panic(err)
	}

	// informing now segment are loaded yet
	z.currentSeg = -1

	// if the file is smaller than 2GB or running on amd64 (so 64 bits) use mmap to read the file
	if uint64(fi.Size()) < chunkSize || runtime.GOARCH == "amd64" {
		mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_READ, syscall.MAP_SHARED)
		if err != nil {
			panic(err)
		}
		z.currentSeg = 0
		z.totalSeg = 1
		z.mmap = mmap
	} else {
		// 32-bit architecture such as Intel's IA-32 can only directly address 4 GiB or smaller portions
		// of files. An even smaller amount of addressible space is available to individual programs
		// typically in the range of 2 to 3 GiB, depending on the operating system kernel.

		z.totalSeg = int(fi.Size()/chunkSize) + 1

		fmt.Printf("Using 32 bits addressing %d segments\n", z.totalSeg)

		// mmap the 1st segment
		err = z.mmapSliceIdx(0)
		if err != nil {
			panic(err)
		}
	}
	err = z.readFileHeaders()
	return &z, err
}

// Return an ordered list of mime types present in the ZIM file
func (z *ZimReader) MimeTypes() []string {
	if len(z.mimeTypeList) != 0 {
		return z.mimeTypeList
	}

	var s []string
	// assume mime list fit in 2k
	b := bytes.NewBuffer(z.getBytesRangeAt(z.mimeListPos, z.mimeListPos+2048))

	for {
		line, err := b.ReadBytes('\x00')
		if err != nil && err != io.EOF {
			panic(err)
		}
		// a line of 1 is a line containing only \x00 and it's the marker for the
		// end of mime types list
		if len(line) == 1 {
			break
		}
		s = append(s, strings.TrimRight(string(line), "\x00"))
	}
	z.mimeTypeList = s
	return s
}

// list all urls contained in a zim file
// note that this is a slow implementation, a real iterator is faster
// but you are not suppose to use this method on big zim files, use indexes
func (z *ZimReader) ListArticles() <-chan *Article {
	ch := make(chan *Article, 10)

	go func() {
		var idx uint32
		// starting at 1 to avoid "con" entry
		var start uint32 = 1

		for idx = start; idx < z.ArticleCount; idx++ {
			offset := z.getURLOffsetAtIdx(idx)
			art := z.getArticleAt(offset)
			if art == nil {
				//TODO: deal with redirect continue
			}
			ch <- art
		}
		close(ch)
	}()
	return ch
}

// return the article at the exact url not using any index this is really slow on big ZIM
func (z *ZimReader) GetPageNoIndex(url string) *Article {
	var idx uint32
	// starting at 1 to avoid "con" entry
	var start uint32 = 1

	art := new(Article)
	for idx = start; idx < z.ArticleCount; idx++ {
		offset := z.getURLOffsetAtIdx(idx)
		art = z.FillArticleAt(art, offset)
		if art.FullURL() == url {
			return art
		}
	}
	return nil
}

// return the article main page if it exists
func (z *ZimReader) GetMainPage() *Article {
	if z.mainPage == 0xffffffff {
		return nil
	}
	return z.getArticleAt(z.getURLOffsetAtIdx(z.mainPage))
}

// Close & cleanup the zimreader
func (z *ZimReader) Close() {
	syscall.Munmap(z.mmap)
	z.f.Close()
}

func (z *ZimReader) String() string {
	fi, err := z.f.Stat()
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("Size: %d, ArticleCount: %d urlPtrPos: 0x%x titlePtrPos: 0x%x mimeListPos: 0x%x clusterPtrPos: 0x%x\nMimeTypes: %v",
		fi.Size(), z.ArticleCount, z.urlPtrPos, z.titlePtrPos, z.mimeListPos, z.clusterPtrPos, z.MimeTypes())
}

// getBytesRangeAt returns bytes from start to end
// it's needed to abstract mmap usages rather than read directly on the mmap slices
func (z *ZimReader) getBytesRangeAt(start, end uint64) []byte {
	if z.totalSeg == 1 {
		return z.mmap[start:end]
	}
	fns := int(start / chunkSize)
	fne := int(end / chunkSize)

	z.Lock()
	defer z.Unlock()

	if fns != z.currentSeg {
		fmt.Println("Need over slice")
		err := z.mmapSliceIdx(fns)
		if err != nil {
			panic(err)
		}
	}

	if fns == fne {
		return z.mmap[start%chunkSize : end%chunkSize]
	}

	fmt.Println("end is on the over file not implemented yet")

	return nil
}

func (z *ZimReader) getByteAt(offset uint64) byte {
	if z.totalSeg == 1 {
		return z.mmap[offset]
	}

	z.Lock()
	defer z.Unlock()

	seg := int(offset / chunkSize)

	if seg != z.currentSeg {
		err := z.mmapSliceIdx(seg)
		if err != nil {
			panic(err)
		}
	}

	return z.mmap[offset%chunkSize]
}

func (z *ZimReader) mmapSliceIdx(idx int) error {
	if idx == z.currentSeg {
		return nil
	}

	if idx >= z.totalSeg {
		return errors.New("idx higher than total segments")
	}

	syscall.Munmap(z.mmap)
	mmap, err := syscall.Mmap(int(z.f.Fd()), int64(idx)*chunkSize, chunkSize, syscall.PROT_READ, syscall.MAP_PRIVATE|syscall.MAP_NORESERVE)
	if err != nil {
		panic(err)
	}
	z.currentSeg = idx
	z.mmap = mmap
	return nil
}

// populate the ZimReader structs with headers
// It is decided to panic on corrupted zim file
func (z *ZimReader) readFileHeaders() error {
	// checking for file type
	v, err := readInt32(z.getBytesRangeAt(0, 0+4))
	if err != nil {
		panic(err)
	}
	if v != zimHeader {
		return errors.New("not a ZIM file")
	}

	// checking for version
	v, err = readInt32(z.getBytesRangeAt(4, 4+4))
	if err != nil {
		panic(err)
	}
	if v != 5 {
		return errors.New("unsupported version, 5 only")
	}

	// checking for articles count
	v, err = readInt32(z.getBytesRangeAt(24, 24+4))
	if err != nil {
		panic(err)
	}
	z.ArticleCount = v

	// checking for cluster count
	v, err = readInt32(z.getBytesRangeAt(28, 28+4))
	if err != nil {
		panic(err)
	}
	z.clusterCount = v

	// checking for urlPtrPos
	vb, err := readInt64(z.getBytesRangeAt(32, 32+8))
	if err != nil {
		panic(err)
	}
	z.urlPtrPos = vb

	// checking for titlePtrPos
	vb, err = readInt64(z.getBytesRangeAt(40, 40+8))
	if err != nil {
		panic(err)
	}
	z.titlePtrPos = vb

	// checking for clusterPtrPos
	vb, err = readInt64(z.getBytesRangeAt(48, 48+8))
	if err != nil {
		panic(err)
	}
	z.clusterPtrPos = vb

	// checking for mimeListPos
	vb, err = readInt64(z.getBytesRangeAt(56, 56+8))
	if err != nil {
		panic(err)
	}
	z.mimeListPos = vb

	// checking for mainPage
	v, err = readInt32(z.getBytesRangeAt(64, 64+4))
	if err != nil {
		panic(err)
	}
	z.mainPage = v

	// checking for layoutPage
	v, err = readInt32(z.getBytesRangeAt(68, 68+4))
	if err != nil {
		panic(err)
	}
	z.layoutPage = v

	z.MimeTypes()
	return err
}

// get the offset in the zim pointing to URL at index pos
func (z *ZimReader) getURLOffsetAtIdx(idx uint32) uint64 {
	offset := z.urlPtrPos + uint64(idx)*8
	v, err := readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	return v
}

// return data at start and end offsets for cluster idx
func (z *ZimReader) getClusterOffsetsAtIdx(idx uint32) (start, end uint64) {
	offset := z.clusterPtrPos + (uint64(idx) * 8)
	start, err := readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	offset = z.clusterPtrPos + (uint64(idx+1) * 8)
	end, err = readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	end--
	return
}
<commit_msg>map the correct size for the last segment<commit_after>package zim

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
	"syscall"
)

const (
	zimHeader = 72173914
)

// ZimReader keep tracks of everything related to ZIM reading
type ZimReader struct {
	f             *os.File
	mmap          []byte
	ArticleCount  uint32
	clusterCount  uint32
	urlPtrPos     uint64
	titlePtrPos   uint64
	clusterPtrPos uint64
	mimeListPos   uint64
	mainPage      uint32
	layoutPage    uint32
	mimeTypeList  []string
	size          int64
	// this mutex is used on 32 bits system only to securise access to one mmap at a time
	sync.Mutex
	// currentSeg indicates which segment is currently mmaped
	currentSeg int
	// totalSeg indicates how many chunks are used to slice the zim file (file size / chunkSize)
	totalSeg int
	// chunkSize needs to be a multiple of os.Getpagesize() for mmap
	segSize int
}

// create a new zim reader
func NewReader(path string) (*ZimReader, error) {
	f, err := os.Open(path)
	if err != nil {
		panic(err)
	}
	z := ZimReader{f: f, mainPage: 0xffffffff, layoutPage: 0xffffffff}

	fi, err := f.Stat()
	if err != nil {
		panic(err)
	}

	z.size = fi.Size()

	// informing now segment are loaded yet
	z.currentSeg = -1

	// if the file is smaller than 2GB or running on amd64 (so 64 bits) use mmap to read the file
	if uint64(z.size) < uint64(z.segSize) || runtime.GOARCH == "amd64" {
		mmap, err := syscall.Mmap(int(f.Fd()), 0, int(z.size), syscall.PROT_READ, syscall.MAP_PRIVATE)
		if err != nil {
			panic(err)
		}
		z.currentSeg = 0
		z.totalSeg = 1
		z.mmap = mmap
	} else {
		// 32-bit architecture such as Intel's IA-32 can only directly address 4 GiB or smaller portions
		// of files. An even smaller amount of addressible space is available to individual programs
		// typically in the range of 2 to 3 GiB, depending on the operating system kernel.

		// we need a multiple of page size smaller than 1 << 31 for mmap segments
		pc := (1 << 31) / os.Getpagesize()
		z.segSize = pc * os.Getpagesize()

		z.totalSeg = int(fi.Size()/int64(z.segSize)) + 1

		fmt.Printf("Using 32 bits addressing %d segments of %d pageSize %d\n", z.totalSeg, z.segSize, os.Getpagesize())

		// mmap the 1st segment
		err = z.mmapSliceIdx(0)
		if err != nil {
			panic(err)
		}
	}
	err = z.readFileHeaders()
	return &z, err
}

// Return an ordered list of mime types present in the ZIM file
func (z *ZimReader) MimeTypes() []string {
	if len(z.mimeTypeList) != 0 {
		return z.mimeTypeList
	}

	var s []string
	// assume mime list fit in 2k
	b := bytes.NewBuffer(z.getBytesRangeAt(z.mimeListPos, z.mimeListPos+2048))

	for {
		line, err := b.ReadBytes('\x00')
		if err != nil && err != io.EOF {
			panic(err)
		}
		// a line of 1 is a line containing only \x00 and it's the marker for the
		// end of mime types list
		if len(line) == 1 {
			break
		}
		s = append(s, strings.TrimRight(string(line), "\x00"))
	}
	z.mimeTypeList = s
	return s
}

// list all urls contained in a zim file
// note that this is a slow implementation, a real iterator is faster
// but you are not suppose to use this method on big zim files, use indexes
func (z *ZimReader) ListArticles() <-chan *Article {
	ch := make(chan *Article, 10)

	go func() {
		var idx uint32
		// starting at 1 to avoid "con" entry
		var start uint32 = 1

		for idx = start; idx < z.ArticleCount; idx++ {
			offset := z.getURLOffsetAtIdx(idx)
			art := z.getArticleAt(offset)
			if art == nil {
				//TODO: deal with redirect continue
			}
			ch <- art
		}
		close(ch)
	}()
	return ch
}

// return the article at the exact url not using any index this is really slow on big ZIM
func (z *ZimReader) GetPageNoIndex(url string) *Article {
	var idx uint32
	// starting at 1 to avoid "con" entry
	var start uint32 = 1

	art := new(Article)
	for idx = start; idx < z.ArticleCount; idx++ {
		offset := z.getURLOffsetAtIdx(idx)
		art = z.FillArticleAt(art, offset)
		if art.FullURL() == url {
			return art
		}
	}
	return nil
}

// return the article main page if it exists
func (z *ZimReader) GetMainPage() *Article {
	if z.mainPage == 0xffffffff {
		return nil
	}
	return z.getArticleAt(z.getURLOffsetAtIdx(z.mainPage))
}

// Close & cleanup the zimreader
func (z *ZimReader) Close() {
	syscall.Munmap(z.mmap)
	z.f.Close()
}

func (z *ZimReader) String() string {
	fi, err := z.f.Stat()
	if err != nil {
		panic(err)
	}
	return fmt.Sprintf("Size: %d, ArticleCount: %d urlPtrPos: 0x%x titlePtrPos: 0x%x mimeListPos: 0x%x clusterPtrPos: 0x%x\nMimeTypes: %v",
		fi.Size(), z.ArticleCount, z.urlPtrPos, z.titlePtrPos, z.mimeListPos, z.clusterPtrPos, z.MimeTypes())
}

// getBytesRangeAt returns bytes from start to end
// it's needed to abstract mmap usages rather than read directly on the mmap slices
func (z *ZimReader) getBytesRangeAt(start, end uint64) []byte {
	if z.totalSeg == 1 {
		return z.mmap[start:end]
	}
	fns := int(start) / z.segSize
	fne := int(end) / z.segSize

	z.Lock()
	defer z.Unlock()

	if fns != z.currentSeg {
		err := z.mmapSliceIdx(fns)
		if err != nil {
			panic(err)
		}
	}

	if fns == fne {
		return z.mmap[start%uint64(z.segSize) : end%uint64(z.segSize)]
	}

	fmt.Println("end is on the over file not implemented yet")

	return nil
}

func (z *ZimReader) getByteAt(offset uint64) byte {
	if z.totalSeg == 1 {
		return z.mmap[offset]
	}

	z.Lock()
	defer z.Unlock()

	seg := int(offset) / z.segSize

	if seg != z.currentSeg {
		err := z.mmapSliceIdx(seg)
		if err != nil {
			panic(err)
		}
	}

	return z.mmap[offset%uint64(z.segSize)]
}

func (z *ZimReader) mmapSliceIdx(idx int) error {
	if idx == z.currentSeg {
		return nil
	}

	if idx >= z.totalSeg {
		return errors.New("idx higher than total segments")
	}

	syscall.Munmap(z.mmap)

	// if this is the last segment mmap the correct size
	mmapSize := z.segSize

	if idx == z.totalSeg-1 {
		mmapSize = int(z.size)%z.segSize + os.Getpagesize()
	}

	mmap, err := syscall.Mmap(int(z.f.Fd()), int64(idx*z.segSize), mmapSize, syscall.PROT_READ, syscall.MAP_SHARED)
	if err != nil {
		panic(err)
	}
	z.currentSeg = idx
	z.mmap = mmap
	return nil
}

// populate the ZimReader structs with headers
// It is decided to panic on corrupted zim file
func (z *ZimReader) readFileHeaders() error {
	// checking for file type
	v, err := readInt32(z.getBytesRangeAt(0, 0+4))
	if err != nil {
		panic(err)
	}
	if v != zimHeader {
		return errors.New("not a ZIM file")
	}

	// checking for version
	v, err = readInt32(z.getBytesRangeAt(4, 4+4))
	if err != nil {
		panic(err)
	}
	if v != 5 {
		return errors.New("unsupported version, 5 only")
	}

	// checking for articles count
	v, err = readInt32(z.getBytesRangeAt(24, 24+4))
	if err != nil {
		panic(err)
	}
	z.ArticleCount = v

	// checking for cluster count
	v, err = readInt32(z.getBytesRangeAt(28, 28+4))
	if err != nil {
		panic(err)
	}
	z.clusterCount = v

	// checking for urlPtrPos
	vb, err := readInt64(z.getBytesRangeAt(32, 32+8))
	if err != nil {
		panic(err)
	}
	z.urlPtrPos = vb

	// checking for titlePtrPos
	vb, err = readInt64(z.getBytesRangeAt(40, 40+8))
	if err != nil {
		panic(err)
	}
	z.titlePtrPos = vb

	// checking for clusterPtrPos
	vb, err = readInt64(z.getBytesRangeAt(48, 48+8))
	if err != nil {
		panic(err)
	}
	z.clusterPtrPos = vb

	// checking for mimeListPos
	vb, err = readInt64(z.getBytesRangeAt(56, 56+8))
	if err != nil {
		panic(err)
	}
	z.mimeListPos = vb

	// checking for mainPage
	v, err = readInt32(z.getBytesRangeAt(64, 64+4))
	if err != nil {
		panic(err)
	}
	z.mainPage = v

	// checking for layoutPage
	v, err = readInt32(z.getBytesRangeAt(68, 68+4))
	if err != nil {
		panic(err)
	}
	z.layoutPage = v

	z.MimeTypes()
	return err
}

// get the offset in the zim pointing to URL at index pos
func (z *ZimReader) getURLOffsetAtIdx(idx uint32) uint64 {
	offset := z.urlPtrPos + uint64(idx)*8
	v, err := readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	return v
}

// return data at start and end offsets for cluster idx
func (z *ZimReader) getClusterOffsetsAtIdx(idx uint32) (start, end uint64) {
	offset := z.clusterPtrPos + (uint64(idx) * 8)
	start, err := readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	offset = z.clusterPtrPos + (uint64(idx+1) * 8)
	end, err = readInt64(z.getBytesRangeAt(offset, offset+8))
	if err != nil {
		panic(err)
	}
	end--
	return
}
<|endoftext|>


<commit_before>// Copyright 2011 Xing Xing <mikespook@gmail.com> All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package worker

import (
    "github.com/miraclesu/gearman-go/common"
    "io"
    "net"
)

// The agent of job server.
type agent struct {
    conn   net.Conn
    worker *Worker
    in     chan []byte
    out    chan *Job
    addr   string
}

// Create the agent of job server.
func newAgent(addr string, worker *Worker) (a *agent, err error) {
    conn, err := net.Dial(common.NETWORK, addr)
    if err != nil {
        return
    }
    a = &agent{
        conn:   conn,
        worker: worker,
        addr:   addr,
        in:     make(chan []byte, common.QUEUE_SIZE),
        out:    make(chan *Job, common.QUEUE_SIZE),
    }
    // reset abilities
    a.WriteJob(newJob(common.REQ, common.RESET_ABILITIES, nil))
    return
}

// outputing loop
func (a *agent) outLoop() {
    ok := true
    var job *Job
    for ok {
        if job, ok = <-a.out; ok {
            if err := a.write(job.Encode()); err != nil {
                a.worker.err(err)
            }
        }
    }
}

// inputing loop
func (a *agent) inLoop() {
    defer func() {
        if r := recover(); r != nil {
            a.worker.err(common.Errorf("Exiting: %s", r))
        }
        close(a.in)
        close(a.out)
        a.worker.removeAgent(a)
    }()
    for a.worker.running {
        a.WriteJob(newJob(common.REQ, common.PRE_SLEEP, nil))
    RESTART:
        // got noop msg and in queue is zero, grab job
        rel, err := a.read()
        if err != nil {
            if err == common.ErrConnection {
                for i := 0; i < 3 && a.worker.running; i++ {
                    if conn, err := net.Dial(common.NETWORK, a.addr); err != nil {
                        a.worker.err(common.Errorf("Reconnection: %d faild", i))
                        continue
                    } else {
                        a.conn = conn
                        goto RESTART
                    }
                }
                a.worker.err(err)
                break
            }
            a.worker.err(err)
            continue
        }
        job, err := decodeJob(rel)
        if err != nil {
            a.worker.err(err)
            continue
        }
        switch job.DataType {
        case common.NOOP:
            a.WriteJob(newJob(common.REQ, common.GRAB_JOB_UNIQ, nil))
        case common.ERROR, common.ECHO_RES, common.JOB_ASSIGN_UNIQ, common.JOB_ASSIGN:
            job.agent = a
            a.worker.in <- job
        }
    }
}

func (a *agent) Close() {
    a.conn.Close()
}

func (a *agent) Work() {
    go a.outLoop()
    go a.inLoop()
}

func (a *agent) readData(length int) (data []byte, err error) {
    n := 0
    buf := make([]byte, common.BUFFER_SIZE)
    // read until data can be unpacked
    for i := length; i > 0 || len(data) < common.PACKET_LEN; i -= n {
        if n, err = a.conn.Read(buf); err != nil {
            if err == io.EOF && n == 0 {
                if data == nil {
                    err = common.ErrConnection
                    return
                }
                return data, nil
            }
            return
        }
        data = append(data, buf[0:n]...)
        if n < common.BUFFER_SIZE {
            break
        }
    }
    return
}

func (a *agent) unpack(data []byte) ([]byte, int, bool) {
    tl := len(data)
    start := 0
    for i := 0; i < tl+1-common.PACKET_LEN; i++ {
        if start+common.PACKET_LEN > tl { // too few data to unpack, read more
            return nil, common.PACKET_LEN, false
        }
        if string(data[start:start+4]) == common.RES_STR {
            l := int(common.BytesToUint32([4]byte{data[start+8],
                data[start+9], data[start+10], data[start+11]}))
            total := l + common.PACKET_LEN
            if total == tl { // data is what we want
                return data, common.PACKET_LEN, true
            } else if total < tl { // data[:total] is what we want, data[total:] is the more
                a.in <- data[total:]
                data = data[start:total]
                return data, common.PACKET_LEN, true
            } else { // ops! It won't be possible.
                return nil, total - tl, false
            }
        } else { // flag was not found, move to next step
            start++
        }
    }
    return nil, common.PACKET_LEN, false
}

func (a *agent) read() (rel []byte, err error) {
    var data []byte
    ok := false
    l := common.PACKET_LEN
    for !ok {
        inlen := len(a.in)
        if inlen > 0 {
            // in queue is not empty
            for i := 0; i < inlen; i++ {
                data = append(data, <-a.in...)
            }
        } else {
            var d []byte
            d, err = a.readData(l)
            if err != nil {
                return
            }
            data = append(data, d...)
        }
        rel, l, ok = a.unpack(data)
    }
    return
}

// Send a job to the job server.
func (a *agent) WriteJob(job *Job) {
    a.out <- job
}

// Internal write the encoded job.
func (a *agent) write(buf []byte) (err error) {
    var n int
    for i := 0; i < len(buf); i += n {
        n, err = a.conn.Write(buf[i:])
        if err != nil {
            return err
        }
    }
    return
}
<commit_msg>fixed the import path<commit_after>// Copyright 2011 Xing Xing <mikespook@gmail.com> All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package worker

import (
    "io"
    "net"
    "github.com/mikespook/gearman-go/common"
)

// The agent of job server.
type agent struct {
    conn   net.Conn
    worker *Worker
    in     chan []byte
    out    chan *Job
    addr   string
}

// Create the agent of job server.
func newAgent(addr string, worker *Worker) (a *agent, err error) {
    conn, err := net.Dial(common.NETWORK, addr)
    if err != nil {
        return
    }
    a = &agent{
        conn:   conn,
        worker: worker,
        addr:   addr,
        in:     make(chan []byte, common.QUEUE_SIZE),
        out:    make(chan *Job, common.QUEUE_SIZE),
    }
    // reset abilities
    a.WriteJob(newJob(common.REQ, common.RESET_ABILITIES, nil))
    return
}

// outputing loop
func (a *agent) outLoop() {
    ok := true
    var job *Job
    for ok {
        if job, ok = <-a.out; ok {
            if err := a.write(job.Encode()); err != nil {
                a.worker.err(err)
            }
        }
    }
}

// inputing loop
func (a *agent) inLoop() {
    defer func() {
        if r := recover(); r != nil {
            a.worker.err(common.Errorf("Exiting: %s", r))
        }
        close(a.in)
        close(a.out)
        a.worker.removeAgent(a)
    }()
    for a.worker.running {
        a.WriteJob(newJob(common.REQ, common.PRE_SLEEP, nil))
    RESTART:
        // got noop msg and in queue is zero, grab job
        rel, err := a.read()
        if err != nil {
            if err == common.ErrConnection {
                for i := 0; i < 3 && a.worker.running; i++ {
                    if conn, err := net.Dial(common.NETWORK, a.addr); err != nil {
                        a.worker.err(common.Errorf("Reconnection: %d faild", i))
                        continue
                    } else {
                        a.conn = conn
                        goto RESTART
                    }
                }
                a.worker.err(err)
                break
            }
            a.worker.err(err)
            continue
        }
        job, err := decodeJob(rel)
        if err != nil {
            a.worker.err(err)
            continue
        }
        switch job.DataType {
        case common.NOOP:
            a.WriteJob(newJob(common.REQ, common.GRAB_JOB_UNIQ, nil))
        case common.ERROR, common.ECHO_RES, common.JOB_ASSIGN_UNIQ, common.JOB_ASSIGN:
            job.agent = a
            a.worker.in <- job
        }
    }
}

func (a *agent) Close() {
    a.conn.Close()
}

func (a *agent) Work() {
    go a.outLoop()
    go a.inLoop()
}

func (a *agent) readData(length int) (data []byte, err error) {
    n := 0
    buf := make([]byte, common.BUFFER_SIZE)
    // read until data can be unpacked
    for i := length; i > 0 || len(data) < common.PACKET_LEN; i -= n {
        if n, err = a.conn.Read(buf); err != nil {
            if err == io.EOF && n == 0 {
                if data == nil {
                    err = common.ErrConnection
                    return
                }
                return data, nil
            }
            return
        }
        data = append(data, buf[0:n]...)
        if n < common.BUFFER_SIZE {
            break
        }
    }
    return
}

func (a *agent) unpack(data []byte) ([]byte, int, bool) {
    tl := len(data)
    start := 0
    for i := 0; i < tl+1-common.PACKET_LEN; i++ {
        if start+common.PACKET_LEN > tl { // too few data to unpack, read more
            return nil, common.PACKET_LEN, false
        }
        if string(data[start:start+4]) == common.RES_STR {
            l := int(common.BytesToUint32([4]byte{data[start+8],
                data[start+9], data[start+10], data[start+11]}))
            total := l + common.PACKET_LEN
            if total == tl { // data is what we want
                return data, common.PACKET_LEN, true
            } else if total < tl { // data[:total] is what we want, data[total:] is the more
                a.in <- data[total:]
                data = data[start:total]
                return data, common.PACKET_LEN, true
            } else { // ops! It won't be possible.
                return nil, total - tl, false
            }
        } else { // flag was not found, move to next step
            start++
        }
    }
    return nil, common.PACKET_LEN, false
}

func (a *agent) read() (rel []byte, err error) {
    var data []byte
    ok := false
    l := common.PACKET_LEN
    for !ok {
        inlen := len(a.in)
        if inlen > 0 {
            // in queue is not empty
            for i := 0; i < inlen; i++ {
                data = append(data, <-a.in...)
            }
        } else {
            var d []byte
            d, err = a.readData(l)
            if err != nil {
                return
            }
            data = append(data, d...)
        }
        rel, l, ok = a.unpack(data)
    }
    return
}

// Send a job to the job server.
func (a *agent) WriteJob(job *Job) {
    a.out <- job
}

// Internal write the encoded job.
func (a *agent) write(buf []byte) (err error) {
    var n int
    for i := 0; i < len(buf); i += n {
        n, err = a.conn.Write(buf[i:])
        if err != nil {
            return err
        }
    }
    return
}
<|endoftext|>


<commit_before>package cache

import (
	"bytes"
	"compress/gzip"
	"errors"
	"fmt"
	"math/rand"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"
)

type DummyCache map[string]Object

func (d DummyCache) Keys() string {
	keyList := make([]string, 0)
	for key, _ := range d {
		keyList = append(keyList, key)
	}

	return strings.Join(keyList, ", ")
}

func (d DummyCache) Get(path string, filler Filler) (Object, error) {
	obj, ok := d[path]
	if !ok {
		return obj, errors.New("Item not found")
	}

	return obj, nil
}

func (d DummyCache) Set(path string, obj Object) error {
	d[path] = obj
	return nil
}

func (d DummyCache) Del(path string) {
	delete(d, path)
}

func unzip(compressed []byte) ([]byte, error) {
	compressedBuf := bytes.NewBuffer(compressed)
	ungz, err := gzip.NewReader(compressedBuf)
	if err != nil {
		return nil, err
	}
	unzipped := bytes.Buffer{}
	_, err = unzipped.ReadFrom(ungz)
	return unzipped.Bytes(), err
}

func Equal(one, two Object) bool {
	// Truncate because disk caches don't store subsecond data.
	oneModTime := one.ModTime.Truncate(time.Second)
	twoModTime := two.ModTime.Truncate(time.Second)
	return bytes.Equal(one.Data, two.Data) && oneModTime == twoModTime
}

func (o Object) String() string {
	return fmt.Sprintf("{%s, %s}", string(o.Data), o.ModTime.String())
}

func simpleObject(length int) Object {
	return Object{make([]byte, length), time.Now()}
}

func TestTestFunctions(t *testing.T) {
	o := Object{[]byte("abc/def"), time.Now()}
	if !Equal(o, o) {
		t.Error("Equal function is broken")
	}
}

func TestCompressAndSet(t *testing.T) {
	c := make(DummyCache)

	key := "abc/def"
	data := []byte("abcdef")
	modTime := time.Now()

	baseObj := Object{data, modTime}

	uncompressed, compressed, err := CompressAndSet(c, key, data, modTime)
	if err != nil {
		t.Errorf("CompressAndSet returned error %s", err)
	}

	if !Equal(uncompressed, baseObj) {
		t.Errorf("Returned uncompressed object \"%s\" not equal to original \"%s\"",
			uncompressed.String(), baseObj.String())
	}

	unzipped, err := unzip(compressed.Data)
	if err != nil {
		t.Errorf("Error decompressing returned data: %s", err)
	}

	unzippedObj := Object{unzipped, compressed.ModTime}
	if !Equal(unzippedObj, baseObj) {
		t.Errorf("Returned compressed object \"%s\" not equal to original \"%s\"",
			unzippedObj.String(), baseObj.String())
	}

	cacheUncompressed, err := c.Get(key, nil)
	if err != nil {
		t.Errorf("Key %s not in cache. Cache contains %s", key, c.Keys())
	}
	if !Equal(cacheUncompressed, baseObj) {
		t.Errorf("Cached uncompressed object \"%s\" not equal to original \"%s\"",
			cacheUncompressed.String(), baseObj.String())
	}

	cacheCompressed, err := c.Get(key+".gz", nil)
	if err != nil {
		t.Errorf("Key %s not in cache. Cache contains %s", key+".gz", c.Keys())
	}
	unzipped, err = unzip(cacheCompressed.Data)
	if err != nil {
		t.Errorf("Error decompressing cached data: %s", err)
	}
	cacheUnzipped := Object{unzipped, cacheCompressed.ModTime}
	if !Equal(cacheUnzipped, baseObj) {
		t.Errorf("Cached compressed object \"%s\" not equal to original \"%s\"",
			cacheUnzipped.String(), baseObj.String())
	}
}

func generatePaths(prefix string, numPaths int) []string {
	paths := make([]string, numPaths)
	for i, _ := range paths {
		paths[i] = fmt.Sprintf("%s%d/%d", prefix, i/10, i)
	}
	return paths
}

func basicCacheTest(t *testing.T, c Cache, objectSize int) {
	obj := simpleObject(objectSize)
	err := c.Set("abc", obj)
	if err != nil {
		t.Errorf("Error adding object: %s", err)
	}
	ret, err := c.Get("abc", nil)
	if err != nil {
		t.Errorf("Error getting object: %s", err)
	}
	if !Equal(obj, ret) {
		t.Errorf("Cache returned %s, expected %s", ret.String(), obj.String())
	}

	c.Del("abc")
	_, err = c.Get("abc", nil)
	if err == nil {
		t.Errorf("Object still in cache after delete")
	}
}

func singleCacheTest(t testing.TB, c Cache, paths []string, objectSize int, allowLoss bool) {
	o := simpleObject(objectSize)

	for _, p := range paths {
		c.Set(p, o)
	}

	for _, p := range paths {
		_, err := c.Get(p, nil)
		if !allowLoss && err != nil {
			t.Errorf("Failed to retrieve path %s: %s.", p, err.Error())
		}
	}
}

func benchmarkSingleCache(b *testing.B, c Cache, objectSize int, allowLoss bool) {
	paths := generatePaths("", b.N)

	b.ResetTimer()
	singleCacheTest(b, c, paths, objectSize, allowLoss)
}

func benchmarkSingleCacheGet(b *testing.B, c Cache, objectSize int, allowLoss bool) {
	o := simpleObject(objectSize)
	paths := generatePaths("", b.N)

	for _, p := range paths {
		c.Set(p, o)
	}

	b.ResetTimer()
	for _, p := range paths {
		_, err := c.Get(p, nil)
		if !allowLoss && err != nil {
			b.Errorf("Failed to retrieve path %s: %s.", p, err.Error())
		}
	}
}

func benchmarkSingleCacheSet(b *testing.B, c Cache, objectSize int, allowLoss bool) {
	o := simpleObject(objectSize)
	paths := generatePaths("", b.N)

	b.ResetTimer()
	for _, p := range paths {
		c.Set(p, o)
	}
}

func parallelSetsLoop(t testing.TB, c Cache, paths []string, objectSize int, filler Filler,
	wg *sync.WaitGroup, start *sync.RWMutex, verify bool) {
	// Wait until the creator unlocks the mutex.
	start.RLock()
	data := []byte(strconv.Itoa(rand.Int()))
	o := Object{data, time.Now()}

	for _, p := range paths {
		c.Set(p, o)
	}

	if verify {
		for _, p := range paths {
			obj, err := c.Get(p, filler)
			if err != nil {
				t.Error("Error retrieving", p, err)
			} else if !Equal(obj, o) {
				throwError := true
				if filler != nil {
					filledObj, _ := filler.Fill(nil, p)
					if Equal(obj, filledObj) {
						throwError = false
					}
				}

				if throwError {
					t.Errorf("Object at path %s was %s, expected %s", p, obj.String(), o.String())
				}
			}
		}
	}

	start.RUnlock()
	wg.Done()
}

func testParallelSets(t testing.TB, c Cache, iterations int, objectSize int, filler Filler,
	numGoroutines int, verify bool) {
	rand.Seed(1)
	wg := &sync.WaitGroup{}
	start := &sync.RWMutex{}
	start.Lock()

	for i := 0; i < numGoroutines; i++ {
		paths := generatePaths(strconv.Itoa(i), iterations)
		wg.Add(1)
		go parallelSetsLoop(t, c, paths, objectSize, filler, wg, start, verify)
	}

	b, ok := t.(*testing.B)
	if ok {
		b.ResetTimer()
	}

	// Unlock the mutex to start all the goroutines running.
	start.Unlock()
	wg.Wait()
}

func benchmarkParallelSets(b *testing.B, c Cache, objectSize int, numGoroutines int) {
	testParallelSets(b, c, b.N/numGoroutines, objectSize, nil, numGoroutines, false)
}

type dummyFiller struct {
	prefix  string
	modTime time.Time
	err     error
}

func (d dummyFiller) Fill(cache Cache, path string) (Object, error) {
	if d.err != nil {
		return Object{}, d.err
	}

	modTime := d.modTime
	if modTime.IsZero() {
		modTime = time.Now()
	}

	data := strings.Join([]string{d.prefix, path}, "-")
	obj := Object{[]byte(data), d.modTime}
	if cache != nil {
		cache.Set(path, obj)
	}
	return obj, nil
}

func testCacheFiller(t *testing.T, c Cache) {
	f := dummyFiller{prefix: "fill", modTime: time.Now()}

	_, err := c.Get("abc", f)
	if err != nil {
		t.Error("Get with Filler returned error", err)
	}

	_, err = c.Get("abc", nil)
	if err != nil {
		t.Error("Get with Filler did not actually fill cache")
	}

	obj := Object{[]byte("test object"), time.Now()}
	err = c.Set("abc", obj)
	if err != nil {
		t.Error("Failed to overwrite object after Filler-backed get")
	}

	obj2, err := c.Get("abc", f)
	if !Equal(obj, obj2) {
		t.Error("Get with Filler should have had cache hit and not used Filler")
	}
}

func testWildcardDelete(t *testing.T, c Cache) {
	paths := []string{"abc", "abd", "subdir/abc", "subdir/abd", "ddd", "subdir/ddd"}

	testPattern := func(pattern string) {
		expected := simpleObject(8)
		for _, p := range paths {
			c.Set(p, expected)
		}

		wildcard := pattern + "*"
		t.Log("Testing delete of", wildcard)
		c.Del(wildcard)
		for _, p := range paths {
			o, err := c.Get(p, nil)
			if strings.HasPrefix(p, pattern) {
				if err == nil {
					t.Errorf("Delete of %s should delete %s", wildcard, p)
				}
			} else {
				if err != nil {
					t.Errorf("Delete of %s should not delete %s", wildcard, p)
				}

				if !Equal(expected, o) {
					t.Errorf("Path %s contained %s, expected %s",
						p, o.String(), expected.String())

				}
			}

		}
	}

	testPattern("abc")
	testPattern("ab")
	testPattern("subdir")
	testPattern("subdir/a")
	testPattern("subdir/abd")
	testPattern("d")
}
<commit_msg>Use bytes.NewReader instead of bytes.NewBuffer<commit_after>package cache

import (
	"bytes"
	"compress/gzip"
	"errors"
	"fmt"
	"math/rand"
	"strconv"
	"strings"
	"sync"
	"testing"
	"time"
)

type DummyCache map[string]Object

func (d DummyCache) Keys() string {
	keyList := make([]string, 0)
	for key, _ := range d {
		keyList = append(keyList, key)
	}

	return strings.Join(keyList, ", ")
}

func (d DummyCache) Get(path string, filler Filler) (Object, error) {
	obj, ok := d[path]
	if !ok {
		return obj, errors.New("Item not found")
	}

	return obj, nil
}

func (d DummyCache) Set(path string, obj Object) error {
	d[path] = obj
	return nil
}

func (d DummyCache) Del(path string) {
	delete(d, path)
}

func unzip(compressed []byte) ([]byte, error) {
	compressedBuf := bytes.NewReader(compressed)
	ungz, err := gzip.NewReader(compressedBuf)
	if err != nil {
		return nil, err
	}
	unzipped := bytes.Buffer{}
	_, err = unzipped.ReadFrom(ungz)
	return unzipped.Bytes(), err
}

func Equal(one, two Object) bool {
	// Truncate because disk caches don't store subsecond data.
	oneModTime := one.ModTime.Truncate(time.Second)
	twoModTime := two.ModTime.Truncate(time.Second)
	return bytes.Equal(one.Data, two.Data) && oneModTime == twoModTime
}

func (o Object) String() string {
	return fmt.Sprintf("{%s, %s}", string(o.Data), o.ModTime.String())
}

func simpleObject(length int) Object {
	return Object{make([]byte, length), time.Now()}
}

func TestTestFunctions(t *testing.T) {
	o := Object{[]byte("abc/def"), time.Now()}
	if !Equal(o, o) {
		t.Error("Equal function is broken")
	}
}

func TestCompressAndSet(t *testing.T) {
	c := make(DummyCache)

	key := "abc/def"
	data := []byte("abcdef")
	modTime := time.Now()

	baseObj := Object{data, modTime}

	uncompressed, compressed, err := CompressAndSet(c, key, data, modTime)
	if err != nil {
		t.Errorf("CompressAndSet returned error %s", err)
	}

	if !Equal(uncompressed, baseObj) {
		t.Errorf("Returned uncompressed object \"%s\" not equal to original \"%s\"",
			uncompressed.String(), baseObj.String())
	}

	unzipped, err := unzip(compressed.Data)
	if err != nil {
		t.Errorf("Error decompressing returned data: %s", err)
	}

	unzippedObj := Object{unzipped, compressed.ModTime}
	if !Equal(unzippedObj, baseObj) {
		t.Errorf("Returned compressed object \"%s\" not equal to original \"%s\"",
			unzippedObj.String(), baseObj.String())
	}

	cacheUncompressed, err := c.Get(key, nil)
	if err != nil {
		t.Errorf("Key %s not in cache. Cache contains %s", key, c.Keys())
	}
	if !Equal(cacheUncompressed, baseObj) {
		t.Errorf("Cached uncompressed object \"%s\" not equal to original \"%s\"",
			cacheUncompressed.String(), baseObj.String())
	}

	cacheCompressed, err := c.Get(key+".gz", nil)
	if err != nil {
		t.Errorf("Key %s not in cache. Cache contains %s", key+".gz", c.Keys())
	}
	unzipped, err = unzip(cacheCompressed.Data)
	if err != nil {
		t.Errorf("Error decompressing cached data: %s", err)
	}
	cacheUnzipped := Object{unzipped, cacheCompressed.ModTime}
	if !Equal(cacheUnzipped, baseObj) {
		t.Errorf("Cached compressed object \"%s\" not equal to original \"%s\"",
			cacheUnzipped.String(), baseObj.String())
	}
}

func generatePaths(prefix string, numPaths int) []string {
	paths := make([]string, numPaths)
	for i, _ := range paths {
		paths[i] = fmt.Sprintf("%s%d/%d", prefix, i/10, i)
	}
	return paths
}

func basicCacheTest(t *testing.T, c Cache, objectSize int) {
	obj := simpleObject(objectSize)
	err := c.Set("abc", obj)
	if err != nil {
		t.Errorf("Error adding object: %s", err)
	}
	ret, err := c.Get("abc", nil)
	if err != nil {
		t.Errorf("Error getting object: %s", err)
	}
	if !Equal(obj, ret) {
		t.Errorf("Cache returned %s, expected %s", ret.String(), obj.String())
	}

	c.Del("abc")
	_, err = c.Get("abc", nil)
	if err == nil {
		t.Errorf("Object still in cache after delete")
	}
}

func singleCacheTest(t testing.TB, c Cache, paths []string, objectSize int, allowLoss bool) {
	o := simpleObject(objectSize)

	for _, p := range paths {
		c.Set(p, o)
	}

	for _, p := range paths {
		_, err := c.Get(p, nil)
		if !allowLoss && err != nil {
			t.Errorf("Failed to retrieve path %s: %s.", p, err.Error())
		}
	}
}

func benchmarkSingleCache(b *testing.B, c Cache, objectSize int, allowLoss bool) {
	paths := generatePaths("", b.N)

	b.ResetTimer()
	singleCacheTest(b, c, paths, objectSize, allowLoss)
}

func benchmarkSingleCacheGet(b *testing.B, c Cache, objectSize int, allowLoss bool) {
	o := simpleObject(objectSize)
	paths := generatePaths("", b.N)

	for _, p := range paths {
		c.Set(p, o)
	}

	b.ResetTimer()
	for _, p := range paths {
		_, err := c.Get(p, nil)
		if !allowLoss && err != nil {
			b.Errorf("Failed to retrieve path %s: %s.", p, err.Error())
		}
	}
}

func benchmarkSingleCacheSet(b *testing.B, c Cache, objectSize int, allowLoss bool) {
	o := simpleObject(objectSize)
	paths := generatePaths("", b.N)

	b.ResetTimer()
	for _, p := range paths {
		c.Set(p, o)
	}
}

func parallelSetsLoop(t testing.TB, c Cache, paths []string, objectSize int, filler Filler,
	wg *sync.WaitGroup, start *sync.RWMutex, verify bool) {
	// Wait until the creator unlocks the mutex.
	start.RLock()
	data := []byte(strconv.Itoa(rand.Int()))
	o := Object{data, time.Now()}

	for _, p := range paths {
		c.Set(p, o)
	}

	if verify {
		for _, p := range paths {
			obj, err := c.Get(p, filler)
			if err != nil {
				t.Error("Error retrieving", p, err)
			} else if !Equal(obj, o) {
				throwError := true
				if filler != nil {
					filledObj, _ := filler.Fill(nil, p)
					if Equal(obj, filledObj) {
						throwError = false
					}
				}

				if throwError {
					t.Errorf("Object at path %s was %s, expected %s", p, obj.String(), o.String())
				}
			}
		}
	}

	start.RUnlock()
	wg.Done()
}

func testParallelSets(t testing.TB, c Cache, iterations int, objectSize int, filler Filler,
	numGoroutines int, verify bool) {
	rand.Seed(1)
	wg := &sync.WaitGroup{}
	start := &sync.RWMutex{}
	start.Lock()

	for i := 0; i < numGoroutines; i++ {
		paths := generatePaths(strconv.Itoa(i), iterations)
		wg.Add(1)
		go parallelSetsLoop(t, c, paths, objectSize, filler, wg, start, verify)
	}

	b, ok := t.(*testing.B)
	if ok {
		b.ResetTimer()
	}

	// Unlock the mutex to start all the goroutines running.
	start.Unlock()
	wg.Wait()
}

func benchmarkParallelSets(b *testing.B, c Cache, objectSize int, numGoroutines int) {
	testParallelSets(b, c, b.N/numGoroutines, objectSize, nil, numGoroutines, false)
}

type dummyFiller struct {
	prefix  string
	modTime time.Time
	err     error
}

func (d dummyFiller) Fill(cache Cache, path string) (Object, error) {
	if d.err != nil {
		return Object{}, d.err
	}

	modTime := d.modTime
	if modTime.IsZero() {
		modTime = time.Now()
	}

	data := strings.Join([]string{d.prefix, path}, "-")
	obj := Object{[]byte(data), d.modTime}
	if cache != nil {
		cache.Set(path, obj)
	}
	return obj, nil
}

func testCacheFiller(t *testing.T, c Cache) {
	f := dummyFiller{prefix: "fill", modTime: time.Now()}

	_, err := c.Get("abc", f)
	if err != nil {
		t.Error("Get with Filler returned error", err)
	}

	_, err = c.Get("abc", nil)
	if err != nil {
		t.Error("Get with Filler did not actually fill cache")
	}

	obj := Object{[]byte("test object"), time.Now()}
	err = c.Set("abc", obj)
	if err != nil {
		t.Error("Failed to overwrite object after Filler-backed get")
	}

	obj2, err := c.Get("abc", f)
	if !Equal(obj, obj2) {
		t.Error("Get with Filler should have had cache hit and not used Filler")
	}
}

func testWildcardDelete(t *testing.T, c Cache) {
	paths := []string{"abc", "abd", "subdir/abc", "subdir/abd", "ddd", "subdir/ddd"}

	testPattern := func(pattern string) {
		expected := simpleObject(8)
		for _, p := range paths {
			c.Set(p, expected)
		}

		wildcard := pattern + "*"
		t.Log("Testing delete of", wildcard)
		c.Del(wildcard)
		for _, p := range paths {
			o, err := c.Get(p, nil)
			if strings.HasPrefix(p, pattern) {
				if err == nil {
					t.Errorf("Delete of %s should delete %s", wildcard, p)
				}
			} else {
				if err != nil {
					t.Errorf("Delete of %s should not delete %s", wildcard, p)
				}

				if !Equal(expected, o) {
					t.Errorf("Path %s contained %s, expected %s",
						p, o.String(), expected.String())

				}
			}

		}
	}

	testPattern("abc")
	testPattern("ab")
	testPattern("subdir")
	testPattern("subdir/a")
	testPattern("subdir/abd")
	testPattern("d")
}
<|endoftext|>


<commit_before>package fuse_test

import (
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"

	"golang.org/x/net/context"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
)

////////////////////////////////////////////////////////////////////////
// minimalFS
////////////////////////////////////////////////////////////////////////

// A minimal fuseutil.FileSystem that can successfully mount but do nothing
// else.
type minimalFS struct {
	fuseutil.NotImplementedFileSystem
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func TestSuccessfulMount(t *testing.T) {
	ctx := context.Background()

	// Set up a temporary directory.
	dir, err := ioutil.TempDir("", "mount_test")
	if err != nil {
		t.Fatal("ioutil.TempDir: %v", err)
	}

	defer os.RemoveAll(dir)

	// Mount.
	fs := &minimalFS{}
	mfs, err := fuse.Mount(
		dir,
		fuseutil.NewFileSystemServer(fs),
		&fuse.MountConfig{})

	if err != nil {
		t.Fatalf("fuse.Mount: %v", err)
	}

	defer func() {
		if err := mfs.Join(ctx); err != nil {
			t.Errorf("Joining: %v", err)
		}
	}()

	defer fuse.Unmount(mfs.Dir())
}

func TestNonEmptyMountPoint(t *testing.T) {
	// Set up a temporary directory.
	dir, err := ioutil.TempDir("", "mount_test")
	if err != nil {
		t.Fatal("ioutil.TempDir: %v", err)
	}

	defer os.RemoveAll(dir)

	// Add a file within it.
	err = ioutil.WriteFile(path.Join(dir, "foo"), []byte{}, 0600)
	if err != nil {
		t.Fatalf("ioutil.WriteFile: %v", err)
	}

	// Attempt to mount.
	fs := &minimalFS{}
	_, err = fuse.Mount(
		dir,
		fuseutil.NewFileSystemServer(fs),
		&fuse.MountConfig{})

	const want = "not empty"
	if err == nil || !strings.Contains(err.Error(), want) {
		t.Errorf("Unexpected error: %v", err)
	}
}
<commit_msg>Fix minimalFS on OS X.<commit_after>package fuse_test

import (
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"

	"golang.org/x/net/context"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
)

////////////////////////////////////////////////////////////////////////
// minimalFS
////////////////////////////////////////////////////////////////////////

// A minimal fuseutil.FileSystem that can successfully mount but do nothing
// else.
type minimalFS struct {
	fuseutil.NotImplementedFileSystem
}

func (fs *minimalFS) StatFS(
	ctx context.Context,
	op *fuseops.StatFSOp) (err error) {
	return
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func TestSuccessfulMount(t *testing.T) {
	ctx := context.Background()

	// Set up a temporary directory.
	dir, err := ioutil.TempDir("", "mount_test")
	if err != nil {
		t.Fatal("ioutil.TempDir: %v", err)
	}

	defer os.RemoveAll(dir)

	// Mount.
	fs := &minimalFS{}
	mfs, err := fuse.Mount(
		dir,
		fuseutil.NewFileSystemServer(fs),
		&fuse.MountConfig{})

	if err != nil {
		t.Fatalf("fuse.Mount: %v", err)
	}

	defer func() {
		if err := mfs.Join(ctx); err != nil {
			t.Errorf("Joining: %v", err)
		}
	}()

	defer fuse.Unmount(mfs.Dir())
}

func TestNonEmptyMountPoint(t *testing.T) {
	// Set up a temporary directory.
	dir, err := ioutil.TempDir("", "mount_test")
	if err != nil {
		t.Fatal("ioutil.TempDir: %v", err)
	}

	defer os.RemoveAll(dir)

	// Add a file within it.
	err = ioutil.WriteFile(path.Join(dir, "foo"), []byte{}, 0600)
	if err != nil {
		t.Fatalf("ioutil.WriteFile: %v", err)
	}

	// Attempt to mount.
	fs := &minimalFS{}
	_, err = fuse.Mount(
		dir,
		fuseutil.NewFileSystemServer(fs),
		&fuse.MountConfig{})

	const want = "not empty"
	if err == nil || !strings.Contains(err.Error(), want) {
		t.Errorf("Unexpected error: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "github.com/nsf/termbox-go"
    "encoding/json"
    "net/http"
    "io/ioutil"
    "strconv"
    "strings"
    "log"
    "sort"
)

type Entries struct {
    Title string
    Year string
    imdbID string
}

type Data struct {
    Search []Entries
}

type MoviesTab struct {
    a *Apollo
    name string
    status string

    movies []Movie
    view string
    sorter string
    offset int
    cursor int
    omdb Data
}

func CreateMoviesTab(a *Apollo) *MoviesTab {
    t := &MoviesTab{
        a: a,
        name: "movies",
        status: "movies",
        view: "watched",
        sorter: "title",
    }

    t.refreshSlice()

    return t
}

func (t *MoviesTab) Name() string {
    return t.name
}

func (t *MoviesTab) Status() string {
    return t.status
}

func (t *MoviesTab) HandleKeyEvent(ev *termbox.Event) bool {
    if len(t.omdb.Search) > 0 {
        indexes := map[rune]int{'0': 0, '1': 1, '2': 2, '3': 3,
                                '4': 4, '5': 5, '6': 6, '7': 7,
                                '8': 8, '9': 9,}
        if index, exist := indexes[ev.Ch]; exist {
            if index < len(t.omdb.Search) {
                t.a.d.Movies[t.index()].Year = t.omdb.Search[index].Year
                t.a.d.Movies[t.index()].Title = t.omdb.Search[index].Title
            }
        }

        t.omdb.Search = t.omdb.Search[:0]
        t.a.d.save()
        t.refreshSlice()

        return true
    }

    switch ev.Ch {
    case '1':
        t.view = "watched"
        t.refreshSlice()
    case '2':
        t.view = "unwatched"
        t.refreshSlice()
    case '3':
        t.view = "all"
        t.refreshSlice()
    case 's':
        t.sort()
    case 'D':
        if len(t.movies) > 0 {
            t.a.d.Movies = append(t.a.d.Movies[:t.index()], t.a.d.Movies[t.index()+1:]...)
            t.refreshSlice()
        }
    case 't':
        t.autoTag()
    }

    switch ev.Key {
    case termbox.KeyArrowUp:
        t.cursor--
        if t.cursor < 0 {
            t.cursor = 0
        } else if t.cursor - t.offset < 0 {
            t.offset--
        }
    case termbox.KeyPgup:
        t.cursor -= 5
        if t.cursor < 0 {
            t.cursor = 0
            t.offset = 0
        } else if t.cursor - t.offset < 0 {
            t.offset -= 5
            if t.offset < 0 {
                t.offset = 0
            }
        }
    case termbox.KeyArrowDown:
        t.cursor++
        if t.cursor > len(t.movies) - 1 {
            t.cursor--
        } else if t.cursor - t.offset > t.a.height - 4 {
            t.offset++
        }
    case termbox.KeyPgdn:
        t.cursor += 5
        if t.cursor > len(t.movies) - 1 {
            t.cursor = len(t.movies) - 1
            if len(t.movies) > t.a.height - 3 {
                t.offset = len(t.movies) - (t.a.height - 3)
            }
        } else if t.cursor - t.offset > t.a.height - 4 {
            t.offset += 5
            if t.cursor - t.offset < t.a.height - 3 {
                t.offset = t.cursor - (t.a.height - 4)
            }
        }
    default:
        return false
    }

    log.Print("cursor: " + strconv.Itoa(t.cursor))

    return true
}

func (t *MoviesTab) Draw() {
    if len(t.omdb.Search) == 0 {
        for j := 0; j < t.a.height - 3; j++ {
            if j < len(t.movies) {
                runes := []rune("[" + t.movies[j + t.offset].Year + "] " + t.movies[j + t.offset].Title)
                for i := 0; i < len(runes); i++ {
                    termbox.SetCell(i + 3, j + 1, runes[i], color['d'], color['d'])
                }
            }
        }

        termbox.SetCell(1, t.cursor - t.offset + 1, '*', color['d'], color['d'])
    } else {
        for j := 0; j < len(t.omdb.Search); j++ {
            runes := []rune(strconv.Itoa(j) + ". [" + t.omdb.Search[j].Year + "] " + t.omdb.Search[j].Title)
            for i := 0; i < len(runes); i++ {
                termbox.SetCell(i, j + 1, runes[i], color['d'], color['d'])
            }
        }
    }
}

func (t *MoviesTab) Query(query string) {
    t.a.d.Movies = append(t.a.d.Movies, Movie{Title: query, State: "Watched"})
    t.a.d.save()
    t.refreshSlice()
}

func (t *MoviesTab) index() int {
    for i := 0; i < len(t.a.d.Movies); i++ {
        if t.a.d.Movies[i].Title == t.movies[t.cursor].Title {
            return i
        }
    }

    return -1
}

func (t *MoviesTab) refreshSlice() {
    t.cursor = 0
    t.offset = 0
    t.movies = t.movies[:0]
    for i := 0; i < len(t.a.d.Movies); i++ {
        if t.a.d.Movies[i].State == "Watched" {
            if t.view == "watched" || t.view == "all" {
                t.movies = append(t.movies, t.a.d.Movies[i])
            }
        } else if t.a.d.Movies[i].State == "Unwatched" {
            if t.view == "unwatched" || t.view == "all" {
                t.movies = append(t.movies, t.a.d.Movies[i])
            }
        }
    }
    t.sort()

    t.status = "movies - " + t.view + " (" + strconv.Itoa(len(t.movies)) + " entries)"
}

func (t *MoviesTab) sort() {
    var titles []string
    for i := 0; i < len(t.movies); i++ {
        titles = append(titles, t.movies[i].Title)
    }
    sort.Strings(titles)

    var movies []Movie
    for j := 0; j < len(titles); j++ {
        for i := 0; i < len(t.movies); i++ {
            if titles[j] == t.movies[i].Title {
                movies = append(movies, t.movies[i])
            }
        }
    }
    t.movies = movies
}

func (t *MoviesTab) autoTag() {
    title := strings.Replace(t.movies[t.cursor].Title, " ", "+", -1)
    url := "http://www.omdbapi.com/?s=" + title + "&type=movie&y=&plot=full&r=json"
    log.Print(url)

    res, err := http.Get(url)
    if err != nil {
        log.Fatal(err)
    }
    defer res.Body.Close()
    body, err := ioutil.ReadAll(res.Body)
    if err != nil {
        log.Fatal(err)
    }

    err = json.Unmarshal(body, &t.omdb)
    if err != nil {
        log.Fatal(err)
    } else {
        for i := 0; i < len(t.omdb.Search); i++ {
            log.Print(t.omdb.Search[i].Title + " - " + t.omdb.Search[i].Year)
        }
    }
}
<commit_msg>Automatically look up omdb after adding a new entry<commit_after>package main

import (
    "github.com/nsf/termbox-go"
    "encoding/json"
    "net/http"
    "io/ioutil"
    "strconv"
    "strings"
    "log"
    "sort"
)

type Entries struct {
    Title string
    Year string
    imdbID string
}

type Data struct {
    Search []Entries
}

type MoviesTab struct {
    a *Apollo
    name string
    status string

    movies []Movie
    view string
    sorter string
    offset int
    cursor int
    omdb Data
}

func CreateMoviesTab(a *Apollo) *MoviesTab {
    t := &MoviesTab{
        a: a,
        name: "movies",
        status: "movies",
        view: "watched",
        sorter: "title",
    }

    t.refreshSlice()

    return t
}

func (t *MoviesTab) Name() string {
    return t.name
}

func (t *MoviesTab) Status() string {
    return t.status
}

func (t *MoviesTab) HandleKeyEvent(ev *termbox.Event) bool {
    if len(t.omdb.Search) > 0 {
        indexes := map[rune]int{'0': 0, '1': 1, '2': 2, '3': 3,
                                '4': 4, '5': 5, '6': 6, '7': 7,
                                '8': 8, '9': 9,}
        if index, exist := indexes[ev.Ch]; exist {
            if index < len(t.omdb.Search) {
                t.a.d.Movies[t.index()].Year = t.omdb.Search[index].Year
                t.a.d.Movies[t.index()].Title = t.omdb.Search[index].Title
            }
        }

        t.omdb.Search = t.omdb.Search[:0]
        t.a.d.save()
        t.refreshSlice()

        return true
    }

    switch ev.Ch {
    case '1':
        t.view = "watched"
        t.refreshSlice()
    case '2':
        t.view = "unwatched"
        t.refreshSlice()
    case '3':
        t.view = "all"
        t.refreshSlice()
    case 's':
        t.sort()
    case 'D':
        if len(t.movies) > 0 {
            t.a.d.Movies = append(t.a.d.Movies[:t.index()], t.a.d.Movies[t.index()+1:]...)
            t.refreshSlice()
        }
    case 't':
        t.autoTag()
    }

    switch ev.Key {
    case termbox.KeyArrowUp:
        t.cursor--
        if t.cursor < 0 {
            t.cursor = 0
        } else if t.cursor - t.offset < 0 {
            t.offset--
        }
    case termbox.KeyPgup:
        t.cursor -= 5
        if t.cursor < 0 {
            t.cursor = 0
            t.offset = 0
        } else if t.cursor - t.offset < 0 {
            t.offset -= 5
            if t.offset < 0 {
                t.offset = 0
            }
        }
    case termbox.KeyArrowDown:
        t.cursor++
        if t.cursor > len(t.movies) - 1 {
            t.cursor--
        } else if t.cursor - t.offset > t.a.height - 4 {
            t.offset++
        }
    case termbox.KeyPgdn:
        t.cursor += 5
        if t.cursor > len(t.movies) - 1 {
            t.cursor = len(t.movies) - 1
            if len(t.movies) > t.a.height - 3 {
                t.offset = len(t.movies) - (t.a.height - 3)
            }
        } else if t.cursor - t.offset > t.a.height - 4 {
            t.offset += 5
            if t.cursor - t.offset < t.a.height - 3 {
                t.offset = t.cursor - (t.a.height - 4)
            }
        }
    default:
        return false
    }

    log.Print("cursor: " + strconv.Itoa(t.cursor))

    return true
}

func (t *MoviesTab) Draw() {
    if len(t.omdb.Search) == 0 {
        for j := 0; j < t.a.height - 3; j++ {
            if j < len(t.movies) {
                runes := []rune("[" + t.movies[j + t.offset].Year + "] " + t.movies[j + t.offset].Title)
                for i := 0; i < len(runes); i++ {
                    termbox.SetCell(i + 3, j + 1, runes[i], color['d'], color['d'])
                }
            }
        }

        termbox.SetCell(1, t.cursor - t.offset + 1, '*', color['d'], color['d'])
    } else {
        for j := 0; j < len(t.omdb.Search); j++ {
            runes := []rune(strconv.Itoa(j) + ". [" + t.omdb.Search[j].Year + "] " + t.omdb.Search[j].Title)
            for i := 0; i < len(runes); i++ {
                termbox.SetCell(i, j + 1, runes[i], color['d'], color['d'])
            }
        }
    }
}

func (t *MoviesTab) Query(query string) {
    t.a.d.Movies = append(t.a.d.Movies, Movie{Title: query, State: "Watched"})
    t.a.d.save()
    t.refreshSlice()

    for i := 0; i < len(t.movies); i++ {
        if t.movies[i].Title == query {
            t.cursor = i
            if t.cursor > t.a.height - 4 {
                t.offset = t.cursor - (t.a.height - 4)
            }
            t.a.inputActive = false
            t.autoTag()
        }
    }
}

func (t *MoviesTab) index() int {
    for i := 0; i < len(t.a.d.Movies); i++ {
        if t.a.d.Movies[i].Title == t.movies[t.cursor].Title {
            return i
        }
    }

    return -1
}

func (t *MoviesTab) refreshSlice() {
    t.cursor = 0
    t.offset = 0
    t.movies = t.movies[:0]
    for i := 0; i < len(t.a.d.Movies); i++ {
        if t.a.d.Movies[i].State == "Watched" {
            if t.view == "watched" || t.view == "all" {
                t.movies = append(t.movies, t.a.d.Movies[i])
            }
        } else if t.a.d.Movies[i].State == "Unwatched" {
            if t.view == "unwatched" || t.view == "all" {
                t.movies = append(t.movies, t.a.d.Movies[i])
            }
        }
    }
    t.sort()

    t.status = "movies - " + t.view + " (" + strconv.Itoa(len(t.movies)) + " entries)"
}

func (t *MoviesTab) sort() {
    var titles []string
    for i := 0; i < len(t.movies); i++ {
        titles = append(titles, t.movies[i].Title)
    }
    sort.Strings(titles)

    var movies []Movie
    for j := 0; j < len(titles); j++ {
        for i := 0; i < len(t.movies); i++ {
            if titles[j] == t.movies[i].Title {
                movies = append(movies, t.movies[i])
            }
        }
    }
    t.movies = movies
}

func (t *MoviesTab) autoTag() {
    title := strings.Replace(t.movies[t.cursor].Title, " ", "+", -1)
    url := "http://www.omdbapi.com/?s=" + title + "&type=movie&y=&plot=full&r=json"
    log.Print(url)

    res, err := http.Get(url)
    if err != nil {
        log.Fatal(err)
    }
    defer res.Body.Close()
    body, err := ioutil.ReadAll(res.Body)
    if err != nil {
        log.Fatal(err)
    }

    err = json.Unmarshal(body, &t.omdb)
    if err != nil {
        log.Fatal(err)
    } else {
        for i := 0; i < len(t.omdb.Search); i++ {
            log.Print(t.omdb.Search[i].Title + " - " + t.omdb.Search[i].Year)
        }
    }
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/arbovm/levenshtein"
	"os"
	"strings"
	"sync"
	"runtime"
)

//GLOBALS
var findings_matches []string
var findings_leven []int
var wg sync.WaitGroup

func abs(x int) int {
	if x < 0 {
		return -x
	} else if x == 0 {
		return 0 // return correctly abs(-0)
	}
	return x
}

func generateHash(path string) {
	inFile, _ := os.Open(path)
	defer inFile.Close()
	scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		s := strings.Replace(scanner.Text(), "/", "", -1)
		partials, num := getPartials(s)
		for i := 0; i < num; i++ {
			addToCache(partials[i], s)
		}
	}
}

func addToCache(spartial string, s string) {
	f, err := os.OpenFile("cache/"+spartial, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.WriteString(s + "\n"); err != nil {
		panic(err)
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
			break
		}
	}
	return false
}

func getPartials(s string) ([]string, int) {
	partials := make([]string, 100000)
	num := 0
	s = strings.ToLower(s)
	s = strings.Replace(s, "/", "", -1)
	s = strings.Replace(s, " ", "", -1)
	s = strings.Replace(s, "the", "", -1)
	s = strings.Replace(s, "by", "", -1)
	s = strings.Replace(s, "dr", "", -1)
	s = strings.Replace(s, "of", "", -1)
	slen := len(s)
	if slen <= 6 {
		partials[num] = "asdf"
		num = num + 1
	} else {
		for i := 0; i <= slen-6; i++ {
			partials[num] = s[i : i+6]
			num = num + 1
		}
	}
	return partials, num
}

func getMatch(s string, path string) (string, int) {
	partials, num := getPartials(s)
	matches := make([]string, 1000000)
	numm := 0
	runtime.GOMAXPROCS(8)
	N := 8
	for i := 0; i < num; i++ {

		inFile, _ := os.Open(path + partials[i])
		defer inFile.Close()
		scanner := bufio.NewScanner(inFile)
		scanner.Split(bufio.ScanLines)

		for scanner.Scan() {
			//if stringInSlice(scanner.Text(),matches) == false { ITS NOT WORTH LOOKING THROUGH DUPLICATES
			matches[numm] = scanner.Text()
			numm = numm + 1
			// }
		}

	}

	matches2 := make([]string, numm)
	matches2 = matches[0:numm]
	findings_leven = make([]int, N)
	findings_matches = make([]string, N)

	wg.Add(N)
	for i := 0; i < N; i++ {
		go search(matches2[i*len(matches2)/N:(i+1)*len(matches2)/N], s, i)
	}
	wg.Wait()

	lowest := 100
	best_index := 0
	for i := 0; i < len(findings_leven); i++ {
		if findings_leven[i] < lowest {
			lowest = findings_leven[i]
			best_index = i
		}
	}

	return findings_matches[best_index],lowest
}

func search(matches []string, target string, process int) {
	defer wg.Done()
	match := "No match"
	target = strings.ToLower(target)
	bestLevenshtein := 1000
	for i := 0; i < len(matches); i++ {
		d := levenshtein.Distance(target, strings.ToLower(matches[i]))
		if d < bestLevenshtein {
			bestLevenshtein = d
			match = matches[i]
		}
	}
	findings_matches[process] = match
	findings_leven[process] = bestLevenshtein
}

func main() {
	if strings.EqualFold(os.Args[1], "help") {
		fmt.Println("Version 1.0 - 4-mer tuples\n")
		fmt.Println("./match-concurrent build <NAME OF WORDLIST> - builds cache/ folder in current directory\n")
		fmt.Println("./match-concurrent 'word or words to match' /directions/to/cache/\n")
	} else if strings.EqualFold(os.Args[1], "build") {
		os.Mkdir("cache", 0775)
		generateHash(os.Args[2])
	} else {
		match,lowest := getMatch(os.Args[1],os.Args[2])
		fmt.Printf("%v|||%v\n", match,lowest)
	}
}
<commit_msg>added indexing to the build<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/arbovm/levenshtein"
	"os"
	"strings"
	"sync"
	"runtime"
	"strconv"
)

//GLOBALS
var findings_matches []string
var findings_leven []int
var wg sync.WaitGroup

func abs(x int) int {
	if x < 0 {
		return -x
	} else if x == 0 {
		return 0 // return correctly abs(-0)
	}
	return x
}

func generateHash(path string) {
	inFile, _ := os.Open(path)
	defer inFile.Close()
	scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)

	lineNum := 0
	for scanner.Scan() {
		lineNum++
		s := strings.Replace(scanner.Text(), "/", "", -1)
		addToCache("keys.list",s)
		partials, num := getPartials(s)
		for i := 0; i < num; i++ {
			addToCache(partials[i], strconv.Itoa(lineNum))
		}
	}
}

func addToCache(spartial string, s string) {
	f, err := os.OpenFile("cache/"+spartial, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	if _, err = f.WriteString(s + "\n"); err != nil {
		panic(err)
	}
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
			break
		}
	}
	return false
}

func getPartials(s string) ([]string, int) {
	partials := make([]string, 100000)
	num := 0
	s = strings.ToLower(s)
	s = strings.Replace(s, "/", "", -1)
	s = strings.Replace(s, " ", "", -1)
	s = strings.Replace(s, "the", "", -1)
	s = strings.Replace(s, "by", "", -1)
	s = strings.Replace(s, "dr", "", -1)
	s = strings.Replace(s, "of", "", -1)
	slen := len(s)
	if slen <= 6 {
		partials[num] = "asdf"
		num = num + 1
	} else {
		for i := 0; i <= slen-6; i++ {
			partials[num] = s[i : i+6]
			num = num + 1
		}
	}
	return partials, num
}

func getMatch(s string, path string) (string, int) {
	partials, num := getPartials(s)
	matches := make([]string, 1000000)
	numm := 0
	runtime.GOMAXPROCS(8)
	N := 8
	for i := 0; i < num; i++ {

		inFile, _ := os.Open(path + partials[i])
		defer inFile.Close()
		scanner := bufio.NewScanner(inFile)
		scanner.Split(bufio.ScanLines)

		for scanner.Scan() {
			//if stringInSlice(scanner.Text(),matches) == false { ITS NOT WORTH LOOKING THROUGH DUPLICATES
			matches[numm] = scanner.Text()
			numm = numm + 1
			// }
		}

	}

	matches2 := make([]string, numm)
	matches2 = matches[0:numm]
	findings_leven = make([]int, N)
	findings_matches = make([]string, N)

	wg.Add(N)
	for i := 0; i < N; i++ {
		go search(matches2[i*len(matches2)/N:(i+1)*len(matches2)/N], s, i)
	}
	wg.Wait()

	lowest := 100
	best_index := 0
	for i := 0; i < len(findings_leven); i++ {
		if findings_leven[i] < lowest {
			lowest = findings_leven[i]
			best_index = i
		}
	}

	return findings_matches[best_index],lowest
}

func search(matches []string, target string, process int) {
	defer wg.Done()
	match := "No match"
	target = strings.ToLower(target)
	bestLevenshtein := 1000
	for i := 0; i < len(matches); i++ {
		d := levenshtein.Distance(target, strings.ToLower(matches[i]))
		if d < bestLevenshtein {
			bestLevenshtein = d
			match = matches[i]
		}
	}
	findings_matches[process] = match
	findings_leven[process] = bestLevenshtein
}

func main() {
	if strings.EqualFold(os.Args[1], "help") {
		fmt.Println("Version 1.0 - 4-mer tuples\n")
		fmt.Println("./match-concurrent build <NAME OF WORDLIST> - builds cache/ folder in current directory\n")
		fmt.Println("./match-concurrent 'word or words to match' /directions/to/cache/\n")
	} else if strings.EqualFold(os.Args[1], "build") {
		os.Mkdir("cache", 0775)
		generateHash(os.Args[2])
	} else {
		match,lowest := getMatch(os.Args[1],os.Args[2])
		fmt.Printf("%v|||%v\n", match,lowest)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"path"
	"reflect"
	"sort"
	"strings"
	"testing"
	"time"

	"k8s.io/kops/cloudmock/aws/mockec2"
	"k8s.io/kops/cmd/kops/util"
	"k8s.io/kops/pkg/testutils"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
)

type LifecycleTestOptions struct {
	t           *testing.T
	SrcDir      string
	Version     string
	ClusterName string

	// Shared is a list of resource ids we expect to be tagged as shared
	Shared []string
}

func (o *LifecycleTestOptions) AddDefaults() {
	if o.Version == "" {
		o.Version = "v1alpha2"
	}
	if o.ClusterName == "" {
		o.ClusterName = strings.Replace(o.SrcDir, "_", "", -1) + ".example.com"
	}

	o.SrcDir = "../../tests/integration/update_cluster/" + o.SrcDir
}

// TestLifecycleMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
func TestLifecycleMinimal(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "minimal",
	})
}

// TestLifecyclePrivateCalico runs the test on a private topology
func TestLifecyclePrivateCalico(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "privatecalico",
	})
}

// TestLifecyclePrivateKopeio runs the test on a private topology, with kopeio networking
func TestLifecyclePrivateKopeio(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "privatekopeio",
		Shared: []string{"nat-12345678"},
	})
}

// TestLifecycleSharedVPC runs the test on a shared VPC
func TestLifecycleSharedVPC(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "shared_vpc",
	})
}

// TestLifecycleSharedSubnet runs the test on a shared subnet
func TestLifecycleSharedSubnet(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "shared_subnet",
		Shared: []string{"subnet-12345678"},
	})
}

// TestLifecyclePrivateSharedSubnet runs the test on a shared subnet with private topology
func TestLifecyclePrivateSharedSubnet(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "private-shared-subnet",
		Shared: []string{"subnet-12345678", "subnet-abcdef"},
	})
}

func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptions, cloud *awsup.MockAWSCloud) {
	t := o.t

	var stdout bytes.Buffer

	inputYAML := "in-" + o.Version + ".yaml"

	factoryOptions := &util.FactoryOptions{}
	factoryOptions.RegistryPath = "memfs://tests"

	factory := util.NewFactory(factoryOptions)

	beforeResources := AllResources(cloud)

	{
		options := &CreateOptions{}
		options.Filenames = []string{path.Join(o.SrcDir, inputYAML)}

		err := RunCreate(factory, &stdout, options)
		if err != nil {
			t.Fatalf("error running %q create: %v", inputYAML, err)
		}
	}

	{
		options := &CreateSecretPublickeyOptions{}
		options.ClusterName = o.ClusterName
		options.Name = "admin"
		options.PublicKeyPath = path.Join(o.SrcDir, "id_rsa.pub")

		err := RunCreateSecretPublicKey(factory, &stdout, options)
		if err != nil {
			t.Fatalf("error running %q create: %v", inputYAML, err)
		}
	}

	{
		options := &UpdateClusterOptions{}
		options.InitDefaults()
		options.MaxTaskDuration = 10 * time.Second
		options.Yes = true

		// We don't test it here, and it adds a dependency on kubectl
		options.CreateKubecfg = false

		_, err := RunUpdateCluster(factory, o.ClusterName, &stdout, options)
		if err != nil {
			t.Fatalf("error running update cluster %q: %v", o.ClusterName, err)
		}
	}

	{
		options := &UpdateClusterOptions{}
		options.InitDefaults()
		options.Target = cloudup.TargetDryRun
		options.MaxTaskDuration = 10 * time.Second

		// We don't test it here, and it adds a dependency on kubectl
		options.CreateKubecfg = false

		results, err := RunUpdateCluster(factory, o.ClusterName, &stdout, options)
		if err != nil {
			t.Fatalf("error running update cluster %q: %v", o.ClusterName, err)
		}

		target := results.Target.(*fi.DryRunTarget)
		if target.HasChanges() {
			var b bytes.Buffer
			if err := target.PrintReport(results.TaskMap, &b); err != nil {
				t.Fatalf("error building report: %v", err)
			}
			t.Fatalf("Target had changes after executing: %v", b.String())
		}
	}

	{
		var ids []string
		for id := range AllResources(cloud) {
			ids = append(ids, id)
		}
		sort.Strings(ids)

		for _, id := range ids {
			tags, err := cloud.GetTags(id)
			if err != nil {
				t.Fatalf("error getting tags for %q: %v", id, err)
			}

			dashIndex := strings.Index(id, "-")
			resource := ""
			if dashIndex == -1 {
				t.Errorf("unknown resource type: %q", id)
			} else {
				resource = id[:dashIndex]
			}

			ownership := tags["kubernetes.io/cluster/"+o.ClusterName]
			if beforeResources[id] != nil {
				expect := ""
				for _, s := range o.Shared {
					if id == s {
						expect = "shared"
					}
				}
				if ownership != expect {
					t.Errorf("unexpected kubernetes.io/cluster/ tag on %q: actual=%q expected=%q", id, ownership, expect)
				}
			} else {
				switch resource {
				case "ami":
				case "sshkey":
					// ignore

				default:
					if ownership == "" {
						t.Errorf("no kubernetes.io/cluster/ tag on %q", id)
					}
				}
			}
		}
	}

	{
		options := &DeleteClusterOptions{}
		options.Yes = true
		options.ClusterName = o.ClusterName
		if err := RunDeleteCluster(factory, &stdout, options); err != nil {
			t.Fatalf("error running delete cluster %q: %v", o.ClusterName, err)
		}
	}
}

// AllResources returns all resources
func AllResources(c *awsup.MockAWSCloud) map[string]interface{} {
	all := make(map[string]interface{})
	for k, v := range c.MockEC2.(*mockec2.MockEC2).All() {
		all[k] = v
	}
	return all
}

func runLifecycleTestAWS(o *LifecycleTestOptions) {
	o.AddDefaults()

	t := o.t

	h := testutils.NewIntegrationTestHarness(o.t)
	defer h.Close()

	h.MockKopsVersion("1.8.1")
	cloud := h.SetupMockAWS()

	var beforeIds []string
	for id := range AllResources(cloud) {
		beforeIds = append(beforeIds, id)
	}
	sort.Strings(beforeIds)

	runLifecycleTest(h, o, cloud)

	var afterIds []string
	for id := range AllResources(cloud) {
		afterIds = append(afterIds, id)
	}
	sort.Strings(afterIds)

	if !reflect.DeepEqual(beforeIds, afterIds) {
		t.Fatalf("resources changed by cluster create / destroy: %v -> %v", beforeIds, afterIds)
	}
}
<commit_msg>Add lifecycle tests for legacy tags<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"path"
	"reflect"
	"sort"
	"strings"
	"testing"
	"time"

	"k8s.io/kops/cloudmock/aws/mockec2"
	"k8s.io/kops/cmd/kops/util"
	"k8s.io/kops/pkg/testutils"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
)

type LifecycleTestOptions struct {
	t           *testing.T
	SrcDir      string
	Version     string
	ClusterName string

	// Shared is a list of resource ids we expect to be tagged as shared
	Shared []string
}

func (o *LifecycleTestOptions) AddDefaults() {
	if o.Version == "" {
		o.Version = "v1alpha2"
	}
	if o.ClusterName == "" {
		o.ClusterName = strings.Replace(o.SrcDir, "_", "", -1) + ".example.com"
	}

	o.SrcDir = "../../tests/integration/update_cluster/" + o.SrcDir
}

// TestLifecycleMinimal runs the test on a minimum configuration, similar to kops create cluster minimal.example.com --zones us-west-1a
func TestLifecycleMinimal(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "minimal",
	})
}

// TestLifecyclePrivateCalico runs the test on a private topology
func TestLifecyclePrivateCalico(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "privatecalico",
	})
}

// TestLifecyclePrivateKopeio runs the test on a private topology, with kopeio networking
func TestLifecyclePrivateKopeio(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "privatekopeio",
		Shared: []string{"nat-12345678"},
	})
}

// TestLifecycleSharedVPC runs the test on a shared VPC
func TestLifecycleSharedVPC(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "shared_vpc",
	})
}

// TestLifecycleSharedSubnet runs the test on a shared subnet
func TestLifecycleSharedSubnet(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "shared_subnet",
		Shared: []string{"subnet-12345678"},
	})
}

// TestLifecyclePrivateSharedSubnet runs the test on a shared subnet with private topology
func TestLifecyclePrivateSharedSubnet(t *testing.T) {
	runLifecycleTestAWS(&LifecycleTestOptions{
		t:      t,
		SrcDir: "private-shared-subnet",
		Shared: []string{"subnet-12345678", "subnet-abcdef"},
	})
}

func runLifecycleTest(h *testutils.IntegrationTestHarness, o *LifecycleTestOptions, cloud *awsup.MockAWSCloud) {
	t := o.t

	var stdout bytes.Buffer

	inputYAML := "in-" + o.Version + ".yaml"

	factoryOptions := &util.FactoryOptions{}
	factoryOptions.RegistryPath = "memfs://tests"

	factory := util.NewFactory(factoryOptions)

	beforeResources := AllResources(cloud)

	{
		options := &CreateOptions{}
		options.Filenames = []string{path.Join(o.SrcDir, inputYAML)}

		err := RunCreate(factory, &stdout, options)
		if err != nil {
			t.Fatalf("error running %q create: %v", inputYAML, err)
		}
	}

	{
		options := &CreateSecretPublickeyOptions{}
		options.ClusterName = o.ClusterName
		options.Name = "admin"
		options.PublicKeyPath = path.Join(o.SrcDir, "id_rsa.pub")

		err := RunCreateSecretPublicKey(factory, &stdout, options)
		if err != nil {
			t.Fatalf("error running %q create: %v", inputYAML, err)
		}
	}

	{
		options := &UpdateClusterOptions{}
		options.InitDefaults()
		options.MaxTaskDuration = 10 * time.Second
		options.Yes = true

		// We don't test it here, and it adds a dependency on kubectl
		options.CreateKubecfg = false

		_, err := RunUpdateCluster(factory, o.ClusterName, &stdout, options)
		if err != nil {
			t.Fatalf("error running update cluster %q: %v", o.ClusterName, err)
		}
	}

	{
		options := &UpdateClusterOptions{}
		options.InitDefaults()
		options.Target = cloudup.TargetDryRun
		options.MaxTaskDuration = 10 * time.Second

		// We don't test it here, and it adds a dependency on kubectl
		options.CreateKubecfg = false

		results, err := RunUpdateCluster(factory, o.ClusterName, &stdout, options)
		if err != nil {
			t.Fatalf("error running update cluster %q: %v", o.ClusterName, err)
		}

		target := results.Target.(*fi.DryRunTarget)
		if target.HasChanges() {
			var b bytes.Buffer
			if err := target.PrintReport(results.TaskMap, &b); err != nil {
				t.Fatalf("error building report: %v", err)
			}
			t.Fatalf("Target had changes after executing: %v", b.String())
		}
	}

	{
		var ids []string
		for id := range AllResources(cloud) {
			ids = append(ids, id)
		}
		sort.Strings(ids)

		for _, id := range ids {
			tags, err := cloud.GetTags(id)
			if err != nil {
				t.Fatalf("error getting tags for %q: %v", id, err)
			}

			dashIndex := strings.Index(id, "-")
			resource := ""
			if dashIndex == -1 {
				t.Errorf("unknown resource type: %q", id)
			} else {
				resource = id[:dashIndex]
			}

			legacy := tags["KubernetesCluster"]
			if legacy != "" && legacy != o.ClusterName {
				t.Errorf("unexpected legacy KubernetesCluster tag: actual=%q cluster=%q", legacy, o.ClusterName)
			}

			ownership := tags["kubernetes.io/cluster/"+o.ClusterName]
			if beforeResources[id] != nil {
				expect := ""
				for _, s := range o.Shared {
					if id == s {
						expect = "shared"
					}
				}
				if ownership != expect {
					t.Errorf("unexpected kubernetes.io/cluster/ tag on %q: actual=%q expected=%q", id, ownership, expect)
				}
			} else {
				switch resource {
				case "ami":
				case "sshkey":
					// ignore

				default:
					if ownership == "" {
						t.Errorf("no kubernetes.io/cluster/ tag on %q", id)
					}
					if legacy == "" {
						// We want to deprecate the KubernetesCluster tag, e.g. in IAM
						// but we should probably keep it around for people that may be using it for other purposes
						t.Errorf("no (legacy) KubernetesCluster tag on %q", id)
					}
				}
			}
		}
	}

	{
		options := &DeleteClusterOptions{}
		options.Yes = true
		options.ClusterName = o.ClusterName
		if err := RunDeleteCluster(factory, &stdout, options); err != nil {
			t.Fatalf("error running delete cluster %q: %v", o.ClusterName, err)
		}
	}
}

// AllResources returns all resources
func AllResources(c *awsup.MockAWSCloud) map[string]interface{} {
	all := make(map[string]interface{})
	for k, v := range c.MockEC2.(*mockec2.MockEC2).All() {
		all[k] = v
	}
	return all
}

func runLifecycleTestAWS(o *LifecycleTestOptions) {
	o.AddDefaults()

	t := o.t

	h := testutils.NewIntegrationTestHarness(o.t)
	defer h.Close()

	h.MockKopsVersion("1.8.1")
	cloud := h.SetupMockAWS()

	var beforeIds []string
	for id := range AllResources(cloud) {
		beforeIds = append(beforeIds, id)
	}
	sort.Strings(beforeIds)

	runLifecycleTest(h, o, cloud)

	var afterIds []string
	for id := range AllResources(cloud) {
		afterIds = append(afterIds, id)
	}
	sort.Strings(afterIds)

	if !reflect.DeepEqual(beforeIds, afterIds) {
		t.Fatalf("resources changed by cluster create / destroy: %v -> %v", beforeIds, afterIds)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/olekukonko/tablewriter"
	"github.com/spf13/cobra"
	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/mustload"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/reason"
	"k8s.io/minikube/pkg/minikube/style"
)

var addonListOutput string
var addonPrintDocs bool

// AddonListTemplate represents the addon list template
type AddonListTemplate struct {
	AddonName   string
	AddonStatus string
}

var addonsListCmd = &cobra.Command{
	Use:   "list",
	Short: "Lists all available minikube addons as well as their current statuses (enabled/disabled)",
	Long:  "Lists all available minikube addons as well as their current statuses (enabled/disabled)",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 0 {
			exit.Message(reason.Usage, "usage: minikube addons list")
		}

		var cc *config.ClusterConfig
		if config.ProfileExists(ClusterFlagValue()) {
			_, cc = mustload.Partial(ClusterFlagValue())
		}
		switch strings.ToLower(addonListOutput) {
		case "list":
			printAddonsList(cc, addonPrintDocs)
		case "json":
			printAddonsJSON(cc)
		default:
			exit.Message(reason.Usage, fmt.Sprintf("invalid output format: %s. Valid values: 'list', 'json'", addonListOutput))
		}
	},
}

func init() {
	addonsListCmd.Flags().StringVarP(&addonListOutput, "output", "o", "list", "minikube addons list --output OUTPUT. json, list")
	addonsListCmd.Flags().BoolVarP(&addonPrintDocs, "docs", "d", false, "If true, print web links to addons' documentation.")
	AddonsCmd.AddCommand(addonsListCmd)
}

var iconFromStatus = func(addonStatus bool) string {
	if addonStatus {
		return "✅"
	}
	return "   " // because emoji indentation is different
}

var stringFromStatus = func(addonStatus bool) string {
	if addonStatus {
		return "enabled"
	}
	return "disabled"
}

var printAddonsList = func(cc *config.ClusterConfig, printDocs bool) {
	addonNames := make([]string, 0, len(assets.Addons))
	for addonName := range assets.Addons {
		addonNames = append(addonNames, addonName)
	}
	sort.Strings(addonNames)

	table := tablewriter.NewWriter(os.Stdout)
	table.SetAutoFormatHeaders(true)
	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
	table.SetCenterSeparator("|")

	// Create table header
	var tHeader []string
	if cc == nil {
		tHeader = []string{"Addon Name", "Maintainer"}
	} else {
		tHeader = []string{"Addon Name", "Profile", "Status", "Maintainer"}
	}
	if printDocs {
		tHeader = append(tHeader, "Docs")
	}
	table.SetHeader(tHeader)

	// Create table data
	var tData [][]string
	var temp []string
	for _, addonName := range addonNames {
		addonBundle := assets.Addons[addonName]
		maintainer := addonBundle.Maintainer
		if maintainer == "" {
			maintainer = "3rd party (unknown)"
		}
		docs := addonBundle.Docs
		if docs == "" {
			docs = "n/a"
		}
		if cc == nil {
			temp = []string{addonName, maintainer}
		} else {
			enabled := addonBundle.IsEnabled(cc)
			temp = []string{addonName, cc.Name, fmt.Sprintf("%s %s", stringFromStatus(enabled), iconFromStatus(enabled)), maintainer}
		}
		if printDocs {
			temp = append(temp, docs)
		}
		tData = append(tData, temp)
	}
	table.AppendBulk(tData)

	table.Render()

	v, _, err := config.ListProfiles()
	if err != nil {
		klog.Errorf("list profiles returned error: %v", err)
	}
	if len(v) > 1 {
		out.Styled(style.Tip, "To see addons list for other profiles use: `minikube addons -p name list`")
	}
}

var printAddonsJSON = func(cc *config.ClusterConfig) {
	addonNames := make([]string, 0, len(assets.Addons))
	for addonName := range assets.Addons {
		addonNames = append(addonNames, addonName)
	}
	sort.Strings(addonNames)

	addonsMap := map[string]map[string]interface{}{}

	for _, addonName := range addonNames {
		if cc == nil {
			addonsMap[addonName] = map[string]interface{}{}
			continue
		}

		addonBundle := assets.Addons[addonName]
		enabled := addonBundle.IsEnabled(cc)

		addonsMap[addonName] = map[string]interface{}{
			"Status":  stringFromStatus(enabled),
			"Profile": cc.Name,
		}
	}
	jsonString, _ := json.Marshal(addonsMap)

	out.String(string(jsonString))
}
<commit_msg>Update docs usage output<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/olekukonko/tablewriter"
	"github.com/spf13/cobra"
	"k8s.io/klog/v2"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/mustload"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/reason"
	"k8s.io/minikube/pkg/minikube/style"
)

var addonListOutput string
var addonPrintDocs bool

// AddonListTemplate represents the addon list template
type AddonListTemplate struct {
	AddonName   string
	AddonStatus string
}

var addonsListCmd = &cobra.Command{
	Use:   "list",
	Short: "Lists all available minikube addons as well as their current statuses (enabled/disabled)",
	Long:  "Lists all available minikube addons as well as their current statuses (enabled/disabled)",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) != 0 {
			exit.Message(reason.Usage, "usage: minikube addons list")
		}

		var cc *config.ClusterConfig
		if config.ProfileExists(ClusterFlagValue()) {
			_, cc = mustload.Partial(ClusterFlagValue())
		}
		switch strings.ToLower(addonListOutput) {
		case "list":
			printAddonsList(cc, addonPrintDocs)
		case "json":
			printAddonsJSON(cc)
		default:
			exit.Message(reason.Usage, fmt.Sprintf("invalid output format: %s. Valid values: 'list', 'json'", addonListOutput))
		}
	},
}

func init() {
	addonsListCmd.Flags().StringVarP(&addonListOutput, "output", "o", "list", "minikube addons list --output OUTPUT. json, list")
	addonsListCmd.Flags().BoolVarP(&addonPrintDocs, "docs", "d", false, "If true, print web links to addons' documentation if using --output=list (default).")
	AddonsCmd.AddCommand(addonsListCmd)
}

var iconFromStatus = func(addonStatus bool) string {
	if addonStatus {
		return "✅"
	}
	return "   " // because emoji indentation is different
}

var stringFromStatus = func(addonStatus bool) string {
	if addonStatus {
		return "enabled"
	}
	return "disabled"
}

var printAddonsList = func(cc *config.ClusterConfig, printDocs bool) {
	addonNames := make([]string, 0, len(assets.Addons))
	for addonName := range assets.Addons {
		addonNames = append(addonNames, addonName)
	}
	sort.Strings(addonNames)

	table := tablewriter.NewWriter(os.Stdout)
	table.SetAutoFormatHeaders(true)
	table.SetBorders(tablewriter.Border{Left: true, Top: true, Right: true, Bottom: true})
	table.SetCenterSeparator("|")

	// Create table header
	var tHeader []string
	if cc == nil {
		tHeader = []string{"Addon Name", "Maintainer"}
	} else {
		tHeader = []string{"Addon Name", "Profile", "Status", "Maintainer"}
	}
	if printDocs {
		tHeader = append(tHeader, "Docs")
	}
	table.SetHeader(tHeader)

	// Create table data
	var tData [][]string
	var temp []string
	for _, addonName := range addonNames {
		addonBundle := assets.Addons[addonName]
		maintainer := addonBundle.Maintainer
		if maintainer == "" {
			maintainer = "3rd party (unknown)"
		}
		docs := addonBundle.Docs
		if docs == "" {
			docs = "n/a"
		}
		if cc == nil {
			temp = []string{addonName, maintainer}
		} else {
			enabled := addonBundle.IsEnabled(cc)
			temp = []string{addonName, cc.Name, fmt.Sprintf("%s %s", stringFromStatus(enabled), iconFromStatus(enabled)), maintainer}
		}
		if printDocs {
			temp = append(temp, docs)
		}
		tData = append(tData, temp)
	}
	table.AppendBulk(tData)

	table.Render()

	v, _, err := config.ListProfiles()
	if err != nil {
		klog.Errorf("list profiles returned error: %v", err)
	}
	if len(v) > 1 {
		out.Styled(style.Tip, "To see addons list for other profiles use: `minikube addons -p name list`")
	}
}

var printAddonsJSON = func(cc *config.ClusterConfig) {
	addonNames := make([]string, 0, len(assets.Addons))
	for addonName := range assets.Addons {
		addonNames = append(addonNames, addonName)
	}
	sort.Strings(addonNames)

	addonsMap := map[string]map[string]interface{}{}

	for _, addonName := range addonNames {
		if cc == nil {
			addonsMap[addonName] = map[string]interface{}{}
			continue
		}

		addonBundle := assets.Addons[addonName]
		enabled := addonBundle.IsEnabled(cc)

		addonsMap[addonName] = map[string]interface{}{
			"Status":  stringFromStatus(enabled),
			"Profile": cc.Name,
		}
	}
	jsonString, _ := json.Marshal(addonsMap)

	out.String(string(jsonString))
}
<|endoftext|>


<commit_before>// +build windows
package buf

import (
	"syscall"
)

func checkReadVConstraint(conn syscall.RawConn) (bool, error) {
	var isSocketReady = false
	var reason error
	/*
			In Windows, WSARecv system call only support socket connection.

			It it required to check if the given fd is of a socket type

		    Fix https://github.com/v2ray/v2ray-core/issues/1666

			Additional Information:
			https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsarecv
			https://docs.microsoft.com/en-us/windows/desktop/api/winsock/nf-winsock-getsockopt
			https://docs.microsoft.com/en-us/windows/desktop/WinSock/sol-socket-socket-options

	*/
	err := conn.Control(func(fd uintptr) {
		var val [4]byte
		var le = int32(len(val))
		err := syscall.Getsockopt(syscall.Handle(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, &val[0], &le)
		if err != nil {
			isSocketReady = false
		} else {
			isSocketReady = true
		}
		reason = err
	})

	return isSocketReady, err
}
<commit_msg>Fix codestyle<commit_after>// +build windows
package buf

import (
	"syscall"
)

func checkReadVConstraint(conn syscall.RawConn) (bool, error) {
	var isSocketReady = false
	var reason error
	/*
			In Windows, WSARecv system call only support socket connection.

			It it required to check if the given fd is of a socket type

			Fix https://github.com/v2ray/v2ray-core/issues/1666

			Additional Information:
			https://docs.microsoft.com/en-us/windows/desktop/api/winsock2/nf-winsock2-wsarecv
			https://docs.microsoft.com/en-us/windows/desktop/api/winsock/nf-winsock-getsockopt
			https://docs.microsoft.com/en-us/windows/desktop/WinSock/sol-socket-socket-options

	*/
	err := conn.Control(func(fd uintptr) {
		var val [4]byte
		var le = int32(len(val))
		err := syscall.Getsockopt(syscall.Handle(fd), syscall.SOL_SOCKET, syscall.SO_RCVBUF, &val[0], &le)
		if err != nil {
			isSocketReady = false
		} else {
			isSocketReady = true
		}
		reason = err
	})

	return isSocketReady, err
}
<|endoftext|>