<commit_before>package nat

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	nat "github.com/fd/go-nat"
	logging "github.com/ipfs/go-log"
	goprocess "github.com/jbenet/goprocess"
	periodic "github.com/jbenet/goprocess/periodic"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

var (
	// ErrNoMapping signals no mapping exists for an address
	ErrNoMapping = errors.New("mapping not established")
)

var log = logging.Logger("nat")

// MappingDuration is a default port mapping duration.
// Port mappings are renewed every (MappingDuration / 3)
const MappingDuration = time.Second * 60

// CacheTime is the time a mapping will cache an external address for
const CacheTime = time.Second * 15

// DiscoverNAT looks for a NAT device in the network and
// returns an object that can manage port mappings.
func DiscoverNAT() *NAT {
	nat, err := nat.DiscoverGateway()
	if err != nil {
		log.Debug("DiscoverGateway error:", err)
		return nil
	}
	addr, err := nat.GetDeviceAddress()
	if err != nil {
		log.Debug("DiscoverGateway address error:", err)
	} else {
		log.Debug("DiscoverGateway address:", addr)
	}
	return newNAT(nat)
}

// NAT is an object that manages address port mappings in
// NATs (Network Address Translators). It is a long-running
// service that will periodically renew port mappings,
// and keep an up-to-date list of all the external addresses.
type NAT struct {
	natmu sync.Mutex
	nat   nat.NAT
	proc  goprocess.Process // manages nat mappings lifecycle

	mappingmu sync.RWMutex // guards mappings
	mappings  map[*mapping]struct{}

	Notifier
}

func newNAT(realNAT nat.NAT) *NAT {
	return &NAT{
		nat:      realNAT,
		proc:     goprocess.WithParent(goprocess.Background()),
		mappings: make(map[*mapping]struct{}),
	}
}

// Close shuts down all port mappings. NAT can no longer be used.
func (nat *NAT) Close() error {
	return nat.proc.Close()
}

// Process returns the nat's life-cycle manager, for making it listen
// to close signals.
func (nat *NAT) Process() goprocess.Process {
	return nat.proc
}

// Mappings returns a slice of all NAT mappings
func (nat *NAT) Mappings() []Mapping {
	nat.mappingmu.Lock()
	maps2 := make([]Mapping, 0, len(nat.mappings))
	for m := range nat.mappings {
		maps2 = append(maps2, m)
	}
	nat.mappingmu.Unlock()
	return maps2
}

func (nat *NAT) addMapping(m *mapping) {
	// make mapping automatically close when nat is closed.
	nat.proc.AddChild(m.proc)

	nat.mappingmu.Lock()
	nat.mappings[m] = struct{}{}
	nat.mappingmu.Unlock()
}

func (nat *NAT) rmMapping(m *mapping) {
	nat.mappingmu.Lock()
	delete(nat.mappings, m)
	nat.mappingmu.Unlock()
}

// NewMapping attemps to construct a mapping on protocol and internal port
// It will also periodically renew the mapping until the returned Mapping
// -- or its parent NAT -- is Closed.
//
// May not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) NewMapping(maddr ma.Multiaddr) (Mapping, error) {
	if nat == nil {
		return nil, fmt.Errorf("no nat available")
	}

	network, addr, err := manet.DialArgs(maddr)
	if err != nil {
		return nil, fmt.Errorf("DialArgs failed on addr: %s", maddr.String())
	}

	switch network {
	case "tcp", "tcp4", "tcp6":
		network = "tcp"
	case "udp", "udp4", "udp6":
		network = "udp"
	default:
		return nil, fmt.Errorf("transport not supported by NAT: %s", network)
	}

	intports := strings.Split(addr, ":")[1]
	intport, err := strconv.Atoi(intports)
	if err != nil {
		return nil, err
	}

	m := &mapping{
		nat:     nat,
		proto:   network,
		intport: intport,
		intaddr: maddr,
	}
	m.proc = goprocess.WithTeardown(func() error {
		nat.rmMapping(m)
		return nil
	})
	nat.addMapping(m)

	m.proc.AddChild(periodic.Every(MappingDuration/3, func(worker goprocess.Process) {
		nat.establishMapping(m)
	}))

	// do it once synchronously, so first mapping is done right away, and before exiting,
	// allowing users -- in the optimistic case -- to use results right after.
	nat.establishMapping(m)
	return m, nil
}

func (nat *NAT) establishMapping(m *mapping) {
	oldport := m.ExternalPort()

	log.Debugf("Attempting port map: %s/%d", m.Protocol(), m.InternalPort())
	comment := "libp2p"
	if m.comment != "" {
		comment = "libp2p-" + m.comment
	}

	nat.natmu.Lock()
	newport, err := nat.nat.AddPortMapping(m.Protocol(), m.InternalPort(), comment, MappingDuration)
	if err != nil {
		// Some hardware does not support mappings with timeout, so try that
		newport, err = nat.nat.AddPortMapping(m.Protocol(), m.InternalPort(), comment, 0)
	}
	nat.natmu.Unlock()

	failure := func() {
		m.setExternalPort(0) // clear mapping
		// TODO: log.Event
		log.Warningf("failed to establish port mapping: %s", err)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingFailed(nat, m, oldport, err)
		})

		// we do not close if the mapping failed,
		// because it may work again next time.
	}

	if err != nil || newport == 0 {
		failure()
		return
	}

	m.setExternalPort(newport)
	ext, err := m.ExternalAddr()
	if err != nil {
		log.Debugf("NAT Mapping addr error: %s %s", m.InternalAddr(), err)
		failure()
		return
	}

	log.Debugf("NAT Mapping: %s --> %s", m.InternalAddr(), ext)
	if oldport != 0 && newport != oldport {
		log.Debugf("failed to renew same port mapping: ch %d -> %d", oldport, newport)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingChanged(nat, m, oldport, newport)
		})
	}

	nat.Notifier.notifyAll(func(n Notifiee) {
		n.MappingSuccess(nat, m)
	})
}

// PortMapAddrs attempts to open (and continue to keep open)
// port mappings for given addrs. This function blocks until
// all  addresses have been tried. This allows clients to
// retrieve results immediately after:
//
//   nat.PortMapAddrs(addrs)
//   mapped := nat.ExternalAddrs()
//
// Some may not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) PortMapAddrs(addrs []ma.Multiaddr) {
	// spin off addr mappings independently.
	var wg sync.WaitGroup
	for _, addr := range addrs {
		// do all of them concurrently
		wg.Add(1)
		go func(addr ma.Multiaddr) {
			defer wg.Done()
			nat.NewMapping(addr)
		}(addr)
	}
	wg.Wait()
}

// MappedAddrs returns address mappings NAT believes have been
// successfully established. Unsuccessful mappings are nil. This is:
//
// 		map[internalAddr]externalAddr
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) MappedAddrs() map[ma.Multiaddr]ma.Multiaddr {

	mappings := nat.Mappings()
	addrmap := make(map[ma.Multiaddr]ma.Multiaddr, len(mappings))

	for _, m := range mappings {
		i := m.InternalAddr()
		e, err := m.ExternalAddr()
		if err != nil {
			addrmap[i] = nil
		} else {
			addrmap[i] = e
		}
	}
	return addrmap
}

// ExternalAddrs returns a list of addresses that NAT believes have
// been successfully established. Unsuccessful mappings are omitted,
// so nat.ExternalAddrs() may return less addresses than nat.InternalAddrs().
// To see which addresses are mapped, use nat.MappedAddrs().
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) ExternalAddrs() []ma.Multiaddr {
	mappings := nat.Mappings()
	addrs := make([]ma.Multiaddr, 0, len(mappings))
	for _, m := range mappings {
		a, err := m.ExternalAddr()
		if err != nil {
			continue // this mapping not currently successful.
		}
		addrs = append(addrs, a)
	}
	return addrs
}
<commit_msg>only map *usable* addresses<commit_after>package nat

import (
	"errors"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	nat "github.com/fd/go-nat"
	logging "github.com/ipfs/go-log"
	goprocess "github.com/jbenet/goprocess"
	periodic "github.com/jbenet/goprocess/periodic"
	ma "github.com/multiformats/go-multiaddr"
	manet "github.com/multiformats/go-multiaddr-net"
)

var (
	// ErrNoMapping signals no mapping exists for an address
	ErrNoMapping = errors.New("mapping not established")
)

var log = logging.Logger("nat")

// MappingDuration is a default port mapping duration.
// Port mappings are renewed every (MappingDuration / 3)
const MappingDuration = time.Second * 60

// CacheTime is the time a mapping will cache an external address for
const CacheTime = time.Second * 15

// DiscoverNAT looks for a NAT device in the network and
// returns an object that can manage port mappings.
func DiscoverNAT() *NAT {
	nat, err := nat.DiscoverGateway()
	if err != nil {
		log.Debug("DiscoverGateway error:", err)
		return nil
	}
	addr, err := nat.GetDeviceAddress()
	if err != nil {
		log.Debug("DiscoverGateway address error:", err)
	} else {
		log.Debug("DiscoverGateway address:", addr)
	}
	return newNAT(nat)
}

// NAT is an object that manages address port mappings in
// NATs (Network Address Translators). It is a long-running
// service that will periodically renew port mappings,
// and keep an up-to-date list of all the external addresses.
type NAT struct {
	natmu sync.Mutex
	nat   nat.NAT
	proc  goprocess.Process // manages nat mappings lifecycle

	mappingmu sync.RWMutex // guards mappings
	mappings  map[*mapping]struct{}

	Notifier
}

func newNAT(realNAT nat.NAT) *NAT {
	return &NAT{
		nat:      realNAT,
		proc:     goprocess.WithParent(goprocess.Background()),
		mappings: make(map[*mapping]struct{}),
	}
}

// Close shuts down all port mappings. NAT can no longer be used.
func (nat *NAT) Close() error {
	return nat.proc.Close()
}

// Process returns the nat's life-cycle manager, for making it listen
// to close signals.
func (nat *NAT) Process() goprocess.Process {
	return nat.proc
}

// Mappings returns a slice of all NAT mappings
func (nat *NAT) Mappings() []Mapping {
	nat.mappingmu.Lock()
	maps2 := make([]Mapping, 0, len(nat.mappings))
	for m := range nat.mappings {
		maps2 = append(maps2, m)
	}
	nat.mappingmu.Unlock()
	return maps2
}

func (nat *NAT) addMapping(m *mapping) {
	// make mapping automatically close when nat is closed.
	nat.proc.AddChild(m.proc)

	nat.mappingmu.Lock()
	nat.mappings[m] = struct{}{}
	nat.mappingmu.Unlock()
}

func (nat *NAT) rmMapping(m *mapping) {
	nat.mappingmu.Lock()
	delete(nat.mappings, m)
	nat.mappingmu.Unlock()
}

// NewMapping attemps to construct a mapping on protocol and internal port
// It will also periodically renew the mapping until the returned Mapping
// -- or its parent NAT -- is Closed.
//
// May not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) NewMapping(maddr ma.Multiaddr) (Mapping, error) {
	if nat == nil {
		return nil, fmt.Errorf("no nat available")
	}

	network, addr, err := manet.DialArgs(maddr)
	if err != nil {
		return nil, fmt.Errorf("DialArgs failed on addr: %s", maddr.String())
	}

	var ip net.IP
	switch network {
	case "tcp", "tcp4", "tcp6":
		addr, err := net.ResolveTCPAddr(network, addr)
		if err != nil {
			return nil, err
		}
		ip = addr.IP
		network = "tcp"
	case "udp", "udp4", "udp6":
		addr, err := net.ResolveUDPAddr(network, addr)
		if err != nil {
			return nil, err
		}
		ip = addr.IP
		network = "udp"
	default:
		return nil, fmt.Errorf("transport not supported by NAT: %s", network)
	}

	// XXX: Known limitation: doesn't handle multiple internal addresses.
	// If this applies to you, you can figure it out yourself. Ideally, the
	// NAT library would allow us to handle this case but the "go way"
	// appears to be to just "shrug" at edge-cases.
	if !ip.IsUnspecified() {
		internalAddr, err := nat.nat.GetInternalAddress()
		if err != nil {
			return nil, fmt.Errorf("failed to discover address on nat: %s", err)
		}
		if !ip.Equal(internalAddr) {
			return nil, fmt.Errorf("nat address is %s, refusing to map %s", internalAddr, ip)
		}
	}

	intports := strings.Split(addr, ":")[1]
	intport, err := strconv.Atoi(intports)
	if err != nil {
		return nil, err
	}

	m := &mapping{
		nat:     nat,
		proto:   network,
		intport: intport,
		intaddr: maddr,
	}
	m.proc = goprocess.WithTeardown(func() error {
		nat.rmMapping(m)
		return nil
	})
	nat.addMapping(m)

	m.proc.AddChild(periodic.Every(MappingDuration/3, func(worker goprocess.Process) {
		nat.establishMapping(m)
	}))

	// do it once synchronously, so first mapping is done right away, and before exiting,
	// allowing users -- in the optimistic case -- to use results right after.
	nat.establishMapping(m)
	return m, nil
}

func (nat *NAT) establishMapping(m *mapping) {
	oldport := m.ExternalPort()

	log.Debugf("Attempting port map: %s/%d", m.Protocol(), m.InternalPort())
	comment := "libp2p"
	if m.comment != "" {
		comment = "libp2p-" + m.comment
	}

	nat.natmu.Lock()
	newport, err := nat.nat.AddPortMapping(m.Protocol(), m.InternalPort(), comment, MappingDuration)
	if err != nil {
		// Some hardware does not support mappings with timeout, so try that
		newport, err = nat.nat.AddPortMapping(m.Protocol(), m.InternalPort(), comment, 0)
	}
	nat.natmu.Unlock()

	failure := func() {
		m.setExternalPort(0) // clear mapping
		// TODO: log.Event
		log.Warningf("failed to establish port mapping: %s", err)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingFailed(nat, m, oldport, err)
		})

		// we do not close if the mapping failed,
		// because it may work again next time.
	}

	if err != nil || newport == 0 {
		failure()
		return
	}

	m.setExternalPort(newport)
	ext, err := m.ExternalAddr()
	if err != nil {
		log.Debugf("NAT Mapping addr error: %s %s", m.InternalAddr(), err)
		failure()
		return
	}

	log.Debugf("NAT Mapping: %s --> %s", m.InternalAddr(), ext)
	if oldport != 0 && newport != oldport {
		log.Debugf("failed to renew same port mapping: ch %d -> %d", oldport, newport)
		nat.Notifier.notifyAll(func(n Notifiee) {
			n.MappingChanged(nat, m, oldport, newport)
		})
	}

	nat.Notifier.notifyAll(func(n Notifiee) {
		n.MappingSuccess(nat, m)
	})
}

// PortMapAddrs attempts to open (and continue to keep open)
// port mappings for given addrs. This function blocks until
// all  addresses have been tried. This allows clients to
// retrieve results immediately after:
//
//   nat.PortMapAddrs(addrs)
//   mapped := nat.ExternalAddrs()
//
// Some may not succeed, and mappings may change over time;
// NAT devices may not respect our port requests, and even lie.
// Clients should not store the mapped results, but rather always
// poll our object for the latest mappings.
func (nat *NAT) PortMapAddrs(addrs []ma.Multiaddr) {
	// spin off addr mappings independently.
	var wg sync.WaitGroup
	for _, addr := range addrs {
		// do all of them concurrently
		wg.Add(1)
		go func(addr ma.Multiaddr) {
			defer wg.Done()
			nat.NewMapping(addr)
		}(addr)
	}
	wg.Wait()
}

// MappedAddrs returns address mappings NAT believes have been
// successfully established. Unsuccessful mappings are nil. This is:
//
// 		map[internalAddr]externalAddr
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) MappedAddrs() map[ma.Multiaddr]ma.Multiaddr {

	mappings := nat.Mappings()
	addrmap := make(map[ma.Multiaddr]ma.Multiaddr, len(mappings))

	for _, m := range mappings {
		i := m.InternalAddr()
		e, err := m.ExternalAddr()
		if err != nil {
			addrmap[i] = nil
		} else {
			addrmap[i] = e
		}
	}
	return addrmap
}

// ExternalAddrs returns a list of addresses that NAT believes have
// been successfully established. Unsuccessful mappings are omitted,
// so nat.ExternalAddrs() may return less addresses than nat.InternalAddrs().
// To see which addresses are mapped, use nat.MappedAddrs().
//
// This set of mappings _may not_ be correct, as NAT devices are finicky.
// Consider this with _best effort_ semantics.
func (nat *NAT) ExternalAddrs() []ma.Multiaddr {
	mappings := nat.Mappings()
	addrs := make([]ma.Multiaddr, 0, len(mappings))
	for _, m := range mappings {
		a, err := m.ExternalAddr()
		if err != nil {
			continue // this mapping not currently successful.
		}
		addrs = append(addrs, a)
	}
	return addrs
}
<|endoftext|>


<commit_before>package main

import "fmt"
//import "io"
import "github.com/amarburg/lazyfs"
import "github.com/amarburg/go-quicktime"

var TestUrlRoot = "http://localhost:8080/files/"
var TestMovPath = "CamHD_Vent_Short.mov"

var SparseHttpStoreRoot = "test_files/httpsparse/"

func main() {

  source,err := lazyfs.OpenHttpFSSource(TestUrlRoot)
  if err != nil {
    panic("Couldn't open HttpFSSource")
  }

  store,err := lazyfs.OpenSparseFileFSStore( SparseHttpStoreRoot )
  if store == nil {
    panic("Couldn't open SparesFileFSStore")
  }

  source.SetBackingStore( store )

  file,err := source.Open( TestMovPath )
  if err != nil {
    panic("Couldn't open AlphabetPath")
  }

  // var offset int64 = 0
  // var indent = 0

  sz,_ := file.FileSize()
//  ParseAtom( file, offset, sz, indent )

  tree := quicktime.BuildTree( file, sz )

  quicktime.DumpTree( file, tree )

  moov := tree.FindAtom("moov")
  if moov == nil { panic("Can't find MOOV atom")}

  tracks := moov.FindAtoms("trak")
  if tracks == nil || len(tracks) == 0 { panic("Couldn't find any TRAKs in the MOOV")}
  fmt.Println("Found",len(tracks),"TRAK atoms")

  var track *quicktime.Atom = nil
  for i,t := range tracks {
    fmt.Println(t, t.Type)
    mdia := t.FindAtom("mdia")
    if mdia == nil {
      fmt.Println("No mdia track",i)
      continue
    }

    minf := mdia.FindAtom("minf")
    if minf == nil {
      fmt.Println("No minf track",i)
      continue
    }

    if minf.FindAtom("vmhd") != nil {
      fmt.Println("Found vmhd")
      track = t
      break
    }
  }

  if track == nil { panic("Couldn't identify the Video track")}

  stbl_atom := track.FindAtom("mdia").FindAtom("minf").FindAtom("stbl")
  stbl,_ := quicktime.ParseSTBL( stbl_atom )

  fmt.Println("Found track with video information")

  // Find movie length
  num_frames := stbl.NumFrames()

  fmt.Println("Movie has",num_frames,"frames")

  fmt.Println("Chunk table:")
  for idx,offset := range stbl.Stco.ChunkOffsets {
    fmt.Printf("   %d %20d\n",idx+1, offset )
  }

  //fmt.Println(stbl)

  for sample := 1; sample <= num_frames; sample++ {
      chunk,chunk_start,relasample := stbl.Stsc.SampleChunk( sample )
      fmt.Println("Sample", sample,"is in chunk",chunk,"the",relasample,"'th sample; the chunk starts at sample",chunk_start)

      offset,_ := stbl.SampleOffset( sample )
      fmt.Println("Sample at byte",offset,"in file")

  }

  // Try extracting a frame
  frame := 1
  LoadFrame( frame, stbl, file )
}

func LoadFrame( frame int, stbl quicktime.STBLAtom, file io.ReaderAt ) {

  frame_offset,frame_size,_ := stbl.SampleOffsetSize( frame )

  fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

  buf := make([]byte, frame_size)
  n,err = file.ReadAt( buf, frame_offset )

  if n != frame_size { panic(fmt.Sprintf("Tried to read %d bytes but got %d instead",frame_size,n))}

}
<commit_msg>Successfully decoded an image.<commit_after>package main

import "fmt"
import "io"
import "os"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/goav/avcodec"
import "github.com/amarburg/goav/avutil"
import "github.com/amarburg/goav/swscale"

import "image"
import "image/png"

import "encoding/binary"
import "bytes"

// //#cgo pkg-config: libavcodec
// //#include <libavcodec/avcodec.h>

import "C"
import "unsafe"

var TestUrlRoot = "http://localhost:8080/files/"
var TestMovPath = "CamHD_Vent_Short.mov"

var SparseHttpStoreRoot = "cache/httpsparse/"

func main() {

  source,err := lazyfs.OpenHttpFSSource(TestUrlRoot)
  if err != nil {
    panic("Couldn't open HttpFSSource")
  }

  store,err := lazyfs.OpenSparseFileFSStore( SparseHttpStoreRoot )
  if store == nil {
    panic("Couldn't open SparesFileFSStore")
  }

  source.SetBackingStore( store )

  file,err := source.Open( TestMovPath )
  if err != nil {
    panic("Couldn't open AlphabetPath")
  }

  // var offset int64 = 0
  // var indent = 0

  sz,_ := file.FileSize()
//  ParseAtom( file, offset, sz, indent )

  tree := quicktime.BuildTree( file, sz )

  quicktime.DumpTree( file, tree )

  moov := tree.FindAtom("moov")
  if moov == nil { panic("Can't find MOOV atom")}

  tracks := moov.FindAtoms("trak")
  if tracks == nil || len(tracks) == 0 { panic("Couldn't find any TRAKs in the MOOV")}
  //fmt.Println("Found",len(tracks),"TRAK atoms")

  var track *quicktime.Atom = nil
  for i,t := range tracks {
    mdia := t.FindAtom("mdia")
    if mdia == nil {
      fmt.Println("No mdia track",i)
      continue
    }

    minf := mdia.FindAtom("minf")
    if minf == nil {
      fmt.Println("No minf track",i)
      continue
    }

    if minf.FindAtom("vmhd") != nil {
      track = t
      break
    }
  }

  if track == nil { panic("Couldn't identify the Video track")}

  stbl_atom := track.FindAtom("mdia").FindAtom("minf").FindAtom("stbl")
  stbl,_ := quicktime.ParseSTBL( stbl_atom )

  //fmt.Println("Found track with video information")

  // Find movie length
  num_frames := stbl.NumFrames()

  fmt.Println("Movie has",num_frames,"frames")

  // fmt.Println("Chunk table:")
  // for idx,offset := range stbl.Stco.ChunkOffsets {
  //   fmt.Printf("   %d %20d\n",idx+1, offset )
  // }

  //fmt.Println(stbl)

  // for sample := 1; sample <= num_frames; sample++ {
  //     chunk,chunk_start,relasample := stbl.Stsc.SampleChunk( sample )
  //     fmt.Println("Sample", sample,"is in chunk",chunk,"the",relasample,"'th sample; the chunk starts at sample",chunk_start)
  //
  //     offset,_ := stbl.SampleOffset( sample )
  //     fmt.Println("Sample at byte",offset,"in file")
  //
  //
  // }

  // Try extracting a frame
  frame := 1
  LoadFrame( frame, stbl, file )
}

func LoadFrame( frame int, stbl quicktime.STBLAtom, file io.ReaderAt ) {

  frame_offset,frame_size,_ := stbl.SampleOffsetSize( frame )

  fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

  buf := make([]byte, frame_size)
  n,_ := file.ReadAt( buf, frame_offset )

  if n != frame_size { panic(fmt.Sprintf("Tried to read %d bytes but got %d instead",frame_size,n))}

  avcodec.AvcodecRegisterAll()
  prores := avcodec.AvcodecFindDecoder( avcodec.CodecId(avcodec.AV_CODEC_ID_PRORES) )
  if prores == nil { panic("Couldn't find ProRes codec")}

  if prores.AvCodecIsDecoder() != 1 { panic("Isn't a decoder")}

  ctx := prores.AvcodecAllocContext3()
  if ctx == nil { panic("Couldn't allocate context") }

  res := ctx.AvcodecOpen2(prores,nil)
  if res < 0 { panic(fmt.Sprintf("Couldn't open context (%d)",res))}

  packet := avcodec.AvPacketAlloc()
  packet.AvInitPacket()
  //if packet == nil { panic("Couldn't allocate packet") }

  res = packet.AvPacketFromData( (*uint8)(unsafe.Pointer(&buf[0])), len(buf) )
  if res < 0 { panic(fmt.Sprintf("Couldn't set avpacket data (%d)",res))}

  // And a frame to receive the data
  videoFrame := avutil.AvFrameAlloc()
  if videoFrame == nil { panic("Couldn't allocate destination frame") }

  ctx.Width = 1920
  ctx.Height = 1080

  got_picture := 0
  res  = ctx.AvcodecDecodeVideo2( (*avcodec.Frame)(unsafe.Pointer(videoFrame)), &got_picture, packet )

fmt.Printf("Got picture: %d\n", got_picture)
fmt.Printf("%#v\n",*videoFrame)

  width,height := 1920,1080 //videoFrame.width, videoFrame.height

  if got_picture == 0 { panic(fmt.Sprintf("Didn't get a picture, err = %04x", -res)) }

  fmt.Printf("Image is %d x %d, format %d\n", width, height, int(ctx.Pix_fmt) )

  // Convert frame to RGB
  dest_fmt := int32(avcodec.AV_PIX_FMT_RGBA)
  flags := 0
  ctxtSws := swscale.SwsGetcontext(width, height, swscale.PixelFormat(ctx.Pix_fmt),
                                  width, height, swscale.PixelFormat(dest_fmt),
                                  flags, nil, nil, nil )
  if ctxtSws == nil { panic("Couldn't open swscale context")}

  videoFrameRgb := avutil.AvFrameAlloc()
  if videoFrameRgb == nil { panic("Couldn't allocate destination frame") }

  videoFrameRgb.Width = ctx.Width
  videoFrameRgb.Height = ctx.Height
  videoFrameRgb.Format = dest_fmt

  fmt.Println(videoFrameRgb )
  avutil.AvFrameGetBuffer( videoFrameRgb, 8)
  fmt.Println(videoFrameRgb )

  swscale.SwsScale( ctxtSws, videoFrame.Data,
                             videoFrame.Linesize,
                             0, height,
                             videoFrameRgb.Data,
                             videoFrameRgb.Linesize )

fmt.Printf("%#v\n",*videoFrameRgb)

  // Convert videoFrameRgb to Go image?
  img := image.NewRGBA( image.Rect(0,0,width,height) )

  img_filename := fmt.Sprintf("frame%06d.png", frame)
  img_file,err := os.Create(img_filename)
  if err != nil { panic(fmt.Sprintf("Error creating png %s: %s", img_filename, err.Error()))}

  rgb_data :=  C.GoBytes(unsafe.Pointer(videoFrameRgb.Data[0]), C.int(videoFrameRgb.Width * videoFrameRgb.Height * 4))
  //pixels := make([]byte, videoFrameRgb.Width * videoFrameRgb.Height * 4 )

  reader := bytes.NewReader( rgb_data )
  err = binary.Read( reader, binary.LittleEndian, &img.Pix)

  if err != nil { panic(fmt.Sprintf("error on binary read: %s", err.Error() ))}

  err = png.Encode( img_file, img )
  if err != nil { panic(fmt.Sprintf("Error writing png %s: %s", img_filename, err.Error()))}

}
<|endoftext|>


<commit_before>package main

import "fmt"
import "io"
import "os"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/goav/avcodec"
import "github.com/amarburg/goav/avutil"
import "github.com/amarburg/goav/swscale"

import "image"
import "image/png"

import "encoding/binary"
import "bytes"

// //#cgo pkg-config: libavcodec
// //#include <libavcodec/avcodec.h>

import "C"
import "unsafe"

var TestUrlRoot = "http://localhost:8080/files/"
var TestMovPath = "CamHD_Vent_Short.mov"

var SparseHttpStoreRoot = "cache/httpsparse/"

func main() {

  source,err := lazyfs.OpenHttpFSSource(TestUrlRoot)
  if err != nil {
    panic("Couldn't open HttpFSSource")
  }

  store,err := lazyfs.OpenSparseFileFSStore( SparseHttpStoreRoot )
  if store == nil {
    panic("Couldn't open SparesFileFSStore")
  }

  source.SetBackingStore( store )

  file,err := source.Open( TestMovPath )
  if err != nil {
    panic("Couldn't open AlphabetPath")
  }

  // var offset int64 = 0
  // var indent = 0

  sz,_ := file.FileSize()
//  ParseAtom( file, offset, sz, indent )

  tree := quicktime.BuildTree( file, sz )

  quicktime.DumpTree( file, tree )

  moov := tree.FindAtom("moov")
  if moov == nil { panic("Can't find MOOV atom")}

  tracks := moov.FindAtoms("trak")
  if tracks == nil || len(tracks) == 0 { panic("Couldn't find any TRAKs in the MOOV")}
  //fmt.Println("Found",len(tracks),"TRAK atoms")

  var track *quicktime.Atom = nil
  for i,t := range tracks {
    mdia := t.FindAtom("mdia")
    if mdia == nil {
      fmt.Println("No mdia track",i)
      continue
    }

    minf := mdia.FindAtom("minf")
    if minf == nil {
      fmt.Println("No minf track",i)
      continue
    }

    if minf.FindAtom("vmhd") != nil {
      track = t
      break
    }
  }

  if track == nil { panic("Couldn't identify the Video track")}

  stbl_atom := track.FindAtom("mdia").FindAtom("minf").FindAtom("stbl")
  stbl,_ := quicktime.ParseSTBL( stbl_atom )

  //fmt.Println("Found track with video information")

  // Find movie length
  num_frames := stbl.NumFrames()

  fmt.Println("Movie has",num_frames,"frames")

  // fmt.Println("Chunk table:")
  // for idx,offset := range stbl.Stco.ChunkOffsets {
  //   fmt.Printf("   %d %20d\n",idx+1, offset )
  // }

  //fmt.Println(stbl)

  // for sample := 1; sample <= num_frames; sample++ {
  //     chunk,chunk_start,relasample := stbl.Stsc.SampleChunk( sample )
  //     fmt.Println("Sample", sample,"is in chunk",chunk,"the",relasample,"'th sample; the chunk starts at sample",chunk_start)
  //
  //     offset,_ := stbl.SampleOffset( sample )
  //     fmt.Println("Sample at byte",offset,"in file")
  //
  //
  // }

  // Try extracting a frame
  frame := 1
  LoadFrame( frame, stbl, file )
}

func LoadFrame( frame int, stbl quicktime.STBLAtom, file io.ReaderAt ) {

  frame_offset,frame_size,_ := stbl.SampleOffsetSize( frame )

  fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

  buf := make([]byte, frame_size)
  n,_ := file.ReadAt( buf, frame_offset )

  if n != frame_size { panic(fmt.Sprintf("Tried to read %d bytes but got %d instead",frame_size,n))}

  avcodec.AvcodecRegisterAll()
  prores := avcodec.AvcodecFindDecoder( avcodec.CodecId(avcodec.AV_CODEC_ID_PRORES) )
  if prores == nil { panic("Couldn't find ProRes codec")}

  if prores.AvCodecIsDecoder() != 1 { panic("Isn't a decoder")}

  ctx := prores.AvcodecAllocContext3()
  if ctx == nil { panic("Couldn't allocate context") }

  res := ctx.AvcodecOpen2(prores,nil)
  if res < 0 { panic(fmt.Sprintf("Couldn't open context (%d)",res))}

  packet := avcodec.AvPacketAlloc()
  packet.AvInitPacket()
  //if packet == nil { panic("Couldn't allocate packet") }

  res = packet.AvPacketFromData( (*uint8)(unsafe.Pointer(&buf[0])), len(buf) )
  if res < 0 { panic(fmt.Sprintf("Couldn't set avpacket data (%d)",res))}

  // And a frame to receive the data
  videoFrame := avutil.AvFrameAlloc()
  if videoFrame == nil { panic("Couldn't allocate destination frame") }

  ctx.Width = 1920
  ctx.Height = 1080

  got_picture := 0
  res  = ctx.AvcodecDecodeVideo2( (*avcodec.Frame)(unsafe.Pointer(videoFrame)), &got_picture, packet )

fmt.Printf("Got picture: %d\n", got_picture)
fmt.Printf("%#v\n",*videoFrame)

  width,height := 1920,1080 //videoFrame.width, videoFrame.height

  if got_picture == 0 { panic(fmt.Sprintf("Didn't get a picture, err = %04x", -res)) }

  fmt.Printf("Image is %d x %d, format %d\n", width, height, int(ctx.Pix_fmt) )

  // Convert frame to RGB
  dest_fmt := int32(avcodec.AV_PIX_FMT_RGBA)
  flags := 0
  ctxtSws := swscale.SwsGetcontext(width, height, swscale.PixelFormat(ctx.Pix_fmt),
                                  width, height, swscale.PixelFormat(dest_fmt),
                                  flags, nil, nil, nil )
  if ctxtSws == nil { panic("Couldn't open swscale context")}

  videoFrameRgb := avutil.AvFrameAlloc()
  if videoFrameRgb == nil { panic("Couldn't allocate destination frame") }

  videoFrameRgb.Width = ctx.Width
  videoFrameRgb.Height = ctx.Height
  videoFrameRgb.Format = dest_fmt

  fmt.Println(videoFrameRgb )
  avutil.AvFrameGetBuffer( videoFrameRgb, 8)
  fmt.Println(videoFrameRgb )

  swscale.SwsScale( ctxtSws, videoFrame.Data,
                             videoFrame.Linesize,
                             0, height,
                             videoFrameRgb.Data,
                             videoFrameRgb.Linesize )

fmt.Printf("%#v\n",*videoFrameRgb)

  // Convert videoFrameRgb to Go image?
  img := image.NewRGBA( image.Rect(0,0,width,height) )

  img_filename := fmt.Sprintf("frame%06d.png", frame)
  img_file,err := os.Create(img_filename)
  if err != nil { panic(fmt.Sprintf("Error creating png %s: %s", img_filename, err.Error()))}

  rgb_data :=  C.GoBytes(unsafe.Pointer(videoFrameRgb.Data[0]), C.int(videoFrameRgb.Width * videoFrameRgb.Height * 4))
  //pixels := make([]byte, videoFrameRgb.Width * videoFrameRgb.Height * 4 )

  reader := bytes.NewReader( rgb_data )
  err = binary.Read( reader, binary.LittleEndian, &img.Pix)

  if err != nil { panic(fmt.Sprintf("error on binary read: %s", err.Error() ))}

  err = png.Encode( img_file, img )
  if err != nil { panic(fmt.Sprintf("Error writing png %s: %s", img_filename, err.Error()))}

}
<commit_msg>Split the libav decoding into a separate package.<commit_after>package main

import "fmt"
import "io"
import "os"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/go-prores-libav"

import "image/png"

var TestUrlRoot = "http://localhost:8080/files/"
var TestMovPath = "CamHD_Vent_Short.mov"

var SparseHttpStoreRoot = "cache/httpsparse/"

func main() {

  source,err := lazyfs.OpenHttpFSSource(TestUrlRoot)
  if err != nil {
    panic("Couldn't open HttpFSSource")
  }

  store,err := lazyfs.OpenSparseFileFSStore( SparseHttpStoreRoot )
  if store == nil {
    panic("Couldn't open SparesFileFSStore")
  }

  source.SetBackingStore( store )

  file,err := source.Open( TestMovPath )
  if err != nil {
    panic("Couldn't open AlphabetPath")
  }

  // var offset int64 = 0
  // var indent = 0

  sz,_ := file.FileSize()
//  ParseAtom( file, offset, sz, indent )

  tree := quicktime.BuildTree( file, sz )

  quicktime.DumpTree( file, tree )

  moov := tree.FindAtom("moov")
  if moov == nil { panic("Can't find MOOV atom")}

  tracks := moov.FindAtoms("trak")
  if tracks == nil || len(tracks) == 0 { panic("Couldn't find any TRAKs in the MOOV")}
  //fmt.Println("Found",len(tracks),"TRAK atoms")

  var track *quicktime.Atom = nil
  for i,t := range tracks {
    mdia := t.FindAtom("mdia")
    if mdia == nil {
      fmt.Println("No mdia track",i)
      continue
    }

    minf := mdia.FindAtom("minf")
    if minf == nil {
      fmt.Println("No minf track",i)
      continue
    }

    if minf.FindAtom("vmhd") != nil {
      track = t
      break
    }
  }

  if track == nil { panic("Couldn't identify the Video track")}

  trak,err := quicktime.ParseTRAK( track )
  if err != nil { panic(fmt.Sprintf("Unable to parse TRAK atom: %s", err.Error()))}
  stbl := &trak.Mdia.Minf.Stbl          // Just an alias

  //fmt.Println("Found track with video information")

  // Find movie length
  num_frames := stbl.NumFrames()

  fmt.Println("Movie has",num_frames,"frames")

  // fmt.Println("Chunk table:")
  // for idx,offset := range stbl.Stco.ChunkOffsets {
  //   fmt.Printf("   %d %20d\n",idx+1, offset )
  // }

  //fmt.Println(stbl)

  // for sample := 1; sample <= num_frames; sample++ {
  //     chunk,chunk_start,relasample := stbl.Stsc.SampleChunk( sample )
  //     fmt.Println("Sample", sample,"is in chunk",chunk,"the",relasample,"'th sample; the chunk starts at sample",chunk_start)
  //
  //     offset,_ := stbl.SampleOffset( sample )
  //     fmt.Println("Sample at byte",offset,"in file")
  //
  //
  // }

  // Try extracting a frame
  frame := 2
  LoadFrame( frame, trak, file )

}

func LoadFrame( frame int, trak quicktime.TRAKAtom, file io.ReaderAt ) {

  frame_offset,frame_size,_ := trak.Mdia.Minf.Stbl.SampleOffsetSize( frame )

  fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

  buf := make([]byte, frame_size)
  n,_ := file.ReadAt( buf, frame_offset )

  if n != frame_size { panic(fmt.Sprintf("Tried to read %d bytes but got %d instead",frame_size,n))}

  width, height := int(trak.Tkhd.Width), int(trak.Tkhd.Height)
  fmt.Printf("Image is %d x %d", width, height)

  img,err := prores.DecodeProRes( buf, width, height )

  if err != nil { panic(fmt.Sprintf("Error decoding frame: %s", err.Error()))}

  img_filename := fmt.Sprintf("frame%06d.png", frame)
  img_file,err := os.Create(img_filename)
  if err != nil { panic(fmt.Sprintf("Error creating png %s: %s", img_filename, err.Error()))}

  err = png.Encode( img_file, img )
  if err != nil { panic(fmt.Sprintf("Error writing png %s: %s", img_filename, err.Error()))}

}
<|endoftext|>


<commit_before>package crawler

import (
	"bufio"
	"bytes"
	"fmt"
	"testing"

	"golang.org/x/net/html"

	. "github.com/smartystreets/goconvey/convey"
)

const (
	emptyHead = " "
)

func minificationCheck(in string, out string) {
	var buf bytes.Buffer
	w := bufio.NewWriter(&buf)
	node, err := html.Parse(bytes.NewReader([]byte(in)))
	if err == nil {
		So(RunMinification1(node), ShouldEqual, nil)
		err = html.Render(w, node)
		if err == nil {
			err = w.Flush()
			if err == nil {
				So(string(buf.Bytes()), ShouldEqual, out)
			}
		}
	}
	So(err, ShouldEqual, nil)
}

func TestErrorTag(t *testing.T) {
	Convey("Test error tag", t, func() {
		in := ""
		node, err := html.Parse(bytes.NewReader([]byte(in)))
		So(err, ShouldEqual, nil)

		node.FirstChild.Type = html.ErrorNode
		So(RunMinification1(node).Error(), ShouldEqual, ErrMinificationUnexpectedNodeType.Error())
	})
}

func TestDoctype(t *testing.T) {
	Convey("Test doctype", t, func() {
		in := "<!DOCTYPE html>"

		minificationCheck(in, in)
	})
}

func TestRemoveComments(t *testing.T) {
	Convey("Remove comment", t, func() {
		in := `

`

		minificationCheck(in, emptyHead)
	})

	Convey("Remove double comment", t, func() {
		in := `

	`

		minificationCheck(in, emptyHead)
	})
}

func TestFuncRemoveNode(t *testing.T) {
	Convey("One tag inside", t, func() {
		in := `

<form attr="a">
aaa
</form>

`
		out := `

`

		minificationCheck(in, out)
	})

	Convey("Left text", t, func() {
		in := `

pre<form></form>

`
		out := `

pre

`

		minificationCheck(in, out)
	})

	Convey("Left text with space", t, func() {
		in := "\n" +
			"
pre \n \t \r <form></form>
\n" +
			""
		out := `

pre

`

		minificationCheck(in, out)
	})

	Convey("Right text", t, func() {
		in := `

<form></form>post

`
		out := `

 post

`

		minificationCheck(in, out)
	})

	Convey("Right text with space", t, func() {
		in := "\n" +
			"
<form></form> \n \t \r post
\n" +
			""
		out := `

 post

`

		minificationCheck(in, out)
	})

	Convey("Left and right text", t, func() {
		in := `

pre<form></form>post

`
		out := `

pre post

`

		minificationCheck(in, out)
	})

	Convey("Left and right text with space", t, func() {
		in := "\n" +
			"
pre \n \t \r <form></form> \n \t \r post
\n" +
			""
		out := `

pre post

`

		minificationCheck(in, out)
	})

	Convey("Left tag", t, func() {
		in := `

pre
<form></form>

`
		out := `

pre

`

		minificationCheck(in, out)
	})

	Convey("Right tag", t, func() {
		in := `

<form></form>
post

`
		out := `

post

`

		minificationCheck(in, out)
	})

	Convey("Left and right tag", t, func() {
		in := `

pre
<form></form>
post

`
		out := `

pre

post

`

		minificationCheck(in, out)
	})
}

func TestRemoveTags(t *testing.T) {
	tags := []string{
		"",
		"<button>text</button>",
		"<object>text</object>",
		"<select>text</select>",
		"",
		"<param name=\"a\"/>",
		"<embed a=\"1\"></embed>",
		"<form>text</form>",
		"<time>text</time>",
		"",
		"<svg>text</svg>",
		"
",
		"
",
	}
	out := `<html><head></head><body> </body></html>`
	for _, tagName := range tags {
		Convey("Removing tag "+html.EscapeString(tagName), t, func() {
			in := fmt.Sprintf(`
%s
`, tagName)
			minificationCheck(in, out)
		})
	}

	Convey("Removing tag hidden input", t, func() {
		in := `
<input type="hidden" />
`
		minificationCheck(in, out)
	})

	Convey("Not Removing no hidden tag input", t, func() {
		in := `
<input type="hidden1"/>
`
		minificationCheck(in, in)
	})

	Convey("Not Removing tag input without type", t, func() {
		in := `
<input v="val"/>
`
		minificationCheck(in, in)
	})

	Convey("Removing tag wdr", t, func() {
		in := `
pre<wbr>post
`
		out := `
prepost
`
		minificationCheck(in, out)
	})

	Convey("Removing tag wdr between tags", t, func() {
		in := `

pre
<wbr>
post

`
		out := `

pre

post

`
		minificationCheck(in, out)
	})
}

func TestFuncRemoveAttr(t *testing.T) {
	Convey("Attributes are not deleted", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("Left and right attributes are not removed", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("Left attributes are not removed", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("Right attributes are not removed", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("One attribute for remove", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("All attributes for remove", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})
}

func TestRemoveAttrs(t *testing.T) {
	attrs := []string{
		"id",
		"alt",
		"cols",
		"class",
		"title",
		"width",
		"align",
		"style",
		"color",
		"valign",
		"target",
		"height",
		"border",
		"hspace",
		"vspace",
		"bgcolor",
		"onclick",
		"colspan",
		"itemprop",
		"disabled",
		"itemtype",
		"itemscope",
		"data-width",
		"cellspacing",
		"cellpadding",
		"bordercolor",
	}
	for _, attr := range attrs {
		Convey("Removing attribute "+attr, t, func() {
			out := `

text

`
			in := fmt.Sprintf(`

text

`, attr)
			minificationCheck(in, out)
		})
	}
}
<commit_msg>тестирование функции раскрытия тега<commit_after>package crawler

import (
	"bufio"
	"bytes"
	"fmt"
	"testing"

	"golang.org/x/net/html"

	. "github.com/smartystreets/goconvey/convey"
)

const (
	emptyHead = " "
)

func minificationCheck(in string, out string) {
	var buf bytes.Buffer
	w := bufio.NewWriter(&buf)
	node, err := html.Parse(bytes.NewReader([]byte(in)))
	if err == nil {
		So(RunMinification1(node), ShouldEqual, nil)
		err = html.Render(w, node)
		if err == nil {
			err = w.Flush()
			if err == nil {
				So(string(buf.Bytes()), ShouldEqual, out)
			}
		}
	}
	So(err, ShouldEqual, nil)
}

func TestErrorTag(t *testing.T) {
	Convey("Test error tag", t, func() {
		in := ""
		node, err := html.Parse(bytes.NewReader([]byte(in)))
		So(err, ShouldEqual, nil)

		node.FirstChild.Type = html.ErrorNode
		So(RunMinification1(node).Error(), ShouldEqual, ErrMinificationUnexpectedNodeType.Error())
	})
}

func TestDoctype(t *testing.T) {
	Convey("Test doctype", t, func() {
		in := "<!DOCTYPE html>"

		minificationCheck(in, in)
	})
}

func TestRemoveComments(t *testing.T) {
	Convey("Remove comment", t, func() {
		in := `

`

		minificationCheck(in, emptyHead)
	})

	Convey("Remove double comment", t, func() {
		in := `

	`

		minificationCheck(in, emptyHead)
	})
}

func TestFuncRemoveAttr(t *testing.T) {
	Convey("Attributes are not deleted", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("Left and right attributes are not removed", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("Left attributes are not removed", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("Right attributes are not removed", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("One attribute for remove", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})

	Convey("All attributes for remove", t, func() {
		in := `

text

	`
		out := `

text

	`
		minificationCheck(in, out)
	})
}

func TestRemoveAttrs(t *testing.T) {
	attrs := []string{
		"id",
		"alt",
		"cols",
		"class",
		"title",
		"width",
		"align",
		"style",
		"color",
		"valign",
		"target",
		"height",
		"border",
		"hspace",
		"vspace",
		"bgcolor",
		"onclick",
		"colspan",
		"itemprop",
		"disabled",
		"itemtype",
		"itemscope",
		"data-width",
		"cellspacing",
		"cellpadding",
		"bordercolor",
	}
	for _, attr := range attrs {
		Convey("Removing attribute "+attr, t, func() {
			out := `

text

`
			in := fmt.Sprintf(`

text

`, attr)
			minificationCheck(in, out)
		})
	}
}

func TestFuncRemoveNode(t *testing.T) {
	Convey("One tag inside", t, func() {
		in := `

<form attr="a">
aaa
</form>

`
		out := `

`

		minificationCheck(in, out)
	})

	Convey("Left text", t, func() {
		in := `

pre<form></form>

`
		out := `

pre

`

		minificationCheck(in, out)
	})

	Convey("Left text with space", t, func() {
		in := "\n" +
			"
pre \n \t \r <form></form>
\n" +
			""
		out := `

pre

`

		minificationCheck(in, out)
	})

	Convey("Right text", t, func() {
		in := `

<form></form>post

`
		out := `

 post

`

		minificationCheck(in, out)
	})

	Convey("Right text with space", t, func() {
		in := "\n" +
			"
<form></form> \n \t \r post
\n" +
			""
		out := `

 post

`

		minificationCheck(in, out)
	})

	Convey("Left and right text", t, func() {
		in := `

pre<form></form>post

`
		out := `

pre post

`

		minificationCheck(in, out)
	})

	Convey("Left and right text with space", t, func() {
		in := "\n" +
			"
pre \n \t \r <form></form> \n \t \r post
\n" +
			""
		out := `

pre post

`

		minificationCheck(in, out)
	})

	Convey("Left tag", t, func() {
		in := `

pre
<form></form>

`
		out := `

pre

`

		minificationCheck(in, out)
	})

	Convey("Right tag", t, func() {
		in := `

<form></form>
post

`
		out := `

post

`

		minificationCheck(in, out)
	})

	Convey("Left and right tag", t, func() {
		in := `

pre
<form></form>
post

`
		out := `

pre

post

`

		minificationCheck(in, out)
	})
}

func TestRemoveTags(t *testing.T) {
	tags := []string{
		"",
		"<button>text</button>",
		"<object>text</object>",
		"<select>text</select>",
		"",
		"<param name=\"a\"/>",
		"<embed a=\"1\"></embed>",
		"<form>text</form>",
		"<time>text</time>",
		"",
		"<svg>text</svg>",
		"
",
		"
",
	}
	out := `<html><head></head><body> </body></html>`
	for _, tagName := range tags {
		Convey("Removing tag "+html.EscapeString(tagName), t, func() {
			in := fmt.Sprintf(`
%s
`, tagName)
			minificationCheck(in, out)
		})
	}

	Convey("Removing tag hidden input", t, func() {
		in := `
<input type="hidden" />
`
		minificationCheck(in, out)
	})

	Convey("Not Removing no hidden tag input", t, func() {
		in := `
<input type="hidden1"/>
`
		minificationCheck(in, in)
	})

	Convey("Not Removing tag input without type", t, func() {
		in := `
<input v="val"/>
`
		minificationCheck(in, in)
	})

	Convey("Removing tag wdr", t, func() {
		in := `
pre<wbr>post
`
		out := `
prepost
`
		minificationCheck(in, out)
	})

	Convey("Removing tag wdr between tags", t, func() {
		in := `

pre
<wbr>
post

`
		out := `

pre

post

`
		minificationCheck(in, out)
	})
}

func TestFuncOpenNode(t *testing.T) {
	Convey("Empty", t, func() {
		in := `

`
		out := `

`

		minificationCheck(in, out)
	})

	Convey("Text inside", t, func() {
		in := `

itext

`
		out := `

itext

`

		minificationCheck(in, out)
	})

	Convey("One tag inside", t, func() {
		in := `

itext

`
		out := `

itext

`

		minificationCheck(in, out)
	})

	Convey("Left text, right tag inside", t, func() {
		in := `

ipreitext

`
		out := `

ipreitext

`

		minificationCheck(in, out)
	})

	Convey("Left tag, right text inside", t, func() {
		in := `

itextipost

`
		out := `

itextipost

`

		minificationCheck(in, out)
	})

	Convey("Left text, right text inside", t, func() {
		in := `

ipreitextipost

`
		out := `

ipreitextipost

`

		minificationCheck(in, out)
	})

	Convey("Text {text text} text", t, func() {
		in := `

preipreitextipostpost

`
		out := `

preipreitextipostpost

`

		minificationCheck(in, out)
	})

	Convey("Text {tag tag} text", t, func() {
		in := `

preitext1itext2post

`
		out := `

preitext1itext2post

`

		minificationCheck(in, out)
	})

	Convey("Tag {text text} tag", t, func() {
		in := `

pre
ipreitextipost
post

`
		out := `

pre
ipreitextipost
post

`

		minificationCheck(in, out)
	})

	Convey("Tag {tag tag} tag", t, func() {
		in := `

pre
itext1itext2
post

`
		out := `

pre
itext1itext2
post

`

		minificationCheck(in, out)
	})

	Convey("Tag {text} tag", t, func() {
		in := `

pre
itext
post

`
		out := `

pre
itext
post

`

		minificationCheck(in, out)
	})

	Convey("Tag {tag} tag", t, func() {
		in := `

pre
itext
post

`
		out := `

pre
itext
post

`

		minificationCheck(in, out)
	})
}
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package ecs

import (
	"fmt"

	"github.com/awslabs/goformation/v4/cloudformation"
	"github.com/awslabs/goformation/v4/cloudformation/ec2"
	"github.com/awslabs/goformation/v4/cloudformation/ecs"
	"github.com/compose-spec/compose-go/types"
)

func (b *ecsAPIService) createNFSmountIngress(securityGroups []string, project *types.Project, n string, template *cloudformation.Template) error {
	target := securityGroups[0]
	for _, s := range project.Services {
		for _, v := range s.Volumes {
			if v.Source != n {
				continue
			}
			var source string
			for net := range s.Networks {
				network := project.Networks[net]
				if ext, ok := network.Extensions[extensionSecurityGroup]; ok {
					source = ext.(string)
				} else {
					source = networkResourceName(net)
				}
				break
			}
			name := fmt.Sprintf("%sNFSMount%s", s.Name, n)
			template.Resources[name] = &ec2.SecurityGroupIngress{
				Description:           fmt.Sprintf("Allow NFS mount for %s on %s", s.Name, n),
				GroupId:               target,
				SourceSecurityGroupId: cloudformation.Ref(source),
				IpProtocol:            "tcp",
				FromPort:              2049,
				ToPort:                2049,
			}
			service := template.Resources[serviceResourceName(s.Name)].(*ecs.Service)
			service.AWSCloudFormationDependsOn = append(service.AWSCloudFormationDependsOn, name)
		}
	}
	return nil
}
<commit_msg>Normalize volume names<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package ecs

import (
	"fmt"

	"github.com/awslabs/goformation/v4/cloudformation"
	"github.com/awslabs/goformation/v4/cloudformation/ec2"
	"github.com/awslabs/goformation/v4/cloudformation/ecs"
	"github.com/compose-spec/compose-go/types"
)

func (b *ecsAPIService) createNFSmountIngress(securityGroups []string, project *types.Project, n string, template *cloudformation.Template) error {
	target := securityGroups[0]
	for _, s := range project.Services {
		for _, v := range s.Volumes {
			if v.Source != n {
				continue
			}
			var source string
			for net := range s.Networks {
				network := project.Networks[net]
				if ext, ok := network.Extensions[extensionSecurityGroup]; ok {
					source = ext.(string)
				} else {
					source = networkResourceName(net)
				}
				break
			}
			name := fmt.Sprintf("%sNFSMount%s", normalizeResourceName(s.Name), n)
			template.Resources[name] = &ec2.SecurityGroupIngress{
				Description:           fmt.Sprintf("Allow NFS mount for %s on %s", s.Name, n),
				GroupId:               target,
				SourceSecurityGroupId: cloudformation.Ref(source),
				IpProtocol:            "tcp",
				FromPort:              2049,
				ToPort:                2049,
			}
			service := template.Resources[serviceResourceName(s.Name)].(*ecs.Service)
			service.AWSCloudFormationDependsOn = append(service.AWSCloudFormationDependsOn, name)
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package networkdb

import (
	"errors"
	"time"

	"github.com/hashicorp/memberlist"
	"github.com/hashicorp/serf/serf"
)

const broadcastTimeout = 5 * time.Second

type networkEventMessage struct {
	id   string
	node string
	msg  []byte
}

func (m *networkEventMessage) Invalidates(other memberlist.Broadcast) bool {
	otherm := other.(*networkEventMessage)
	return m.id == otherm.id && m.node == otherm.node
}

func (m *networkEventMessage) Message() []byte {
	return m.msg
}

func (m *networkEventMessage) Finished() {
}

func (nDB *NetworkDB) sendNetworkEvent(nid string, event NetworkEvent_Type, ltime serf.LamportTime) error {
	nEvent := NetworkEvent{
		Type:      event,
		LTime:     ltime,
		NodeName:  nDB.config.NodeName,
		NetworkID: nid,
	}

	raw, err := encodeMessage(MessageTypeNetworkEvent, &nEvent)
	if err != nil {
		return err
	}

	nDB.networkBroadcasts.QueueBroadcast(&networkEventMessage{
		msg:  raw,
		id:   nid,
		node: nDB.config.NodeName,
	})
	return nil
}

type nodeEventMessage struct {
	msg    []byte
	notify chan<- struct{}
}

func (m *nodeEventMessage) Invalidates(other memberlist.Broadcast) bool {
	return false
}

func (m *nodeEventMessage) Message() []byte {
	return m.msg
}

func (m *nodeEventMessage) Finished() {
	if m.notify != nil {
		close(m.notify)
	}
}

func (nDB *NetworkDB) sendNodeEvent(event NodeEvent_Type) error {
	nEvent := NodeEvent{
		Type:     event,
		LTime:    nDB.networkClock.Increment(),
		NodeName: nDB.config.NodeName,
	}

	raw, err := encodeMessage(MessageTypeNodeEvent, &nEvent)
	if err != nil {
		return err
	}

	notifyCh := make(chan struct{})
	nDB.nodeBroadcasts.QueueBroadcast(&nodeEventMessage{
		msg:    raw,
		notify: notifyCh,
	})

	// Wait for the broadcast
	select {
	case <-notifyCh:
	case <-time.After(broadcastTimeout):
		return errors.New("timed out broadcasting node event")
	}

	return nil
}

type tableEventMessage struct {
	id    string
	tname string
	key   string
	msg   []byte
	node  string
}

func (m *tableEventMessage) Invalidates(other memberlist.Broadcast) bool {
	otherm := other.(*tableEventMessage)
	return m.id == otherm.id && m.tname == otherm.tname && m.key == otherm.key
}

func (m *tableEventMessage) Message() []byte {
	return m.msg
}

func (m *tableEventMessage) Finished() {
}

func (nDB *NetworkDB) sendTableEvent(event TableEvent_Type, nid string, tname string, key string, entry *entry) error {
	tEvent := TableEvent{
		Type:      event,
		LTime:     entry.ltime,
		NodeName:  nDB.config.NodeName,
		NetworkID: nid,
		TableName: tname,
		Key:       key,
		Value:     entry.value,
	}

	raw, err := encodeMessage(MessageTypeTableEvent, &tEvent)
	if err != nil {
		return err
	}

	var broadcastQ *memberlist.TransmitLimitedQueue
	nDB.RLock()
	thisNodeNetworks, ok := nDB.networks[nDB.config.NodeName]
	if ok {
		// The network may have been removed
		network, networkOk := thisNodeNetworks[nid]
		if !networkOk {
			nDB.RUnlock()
			return nil
		}

		broadcastQ = network.tableBroadcasts
	}
	nDB.RUnlock()

	// The network may have been removed
	if broadcastQ == nil {
		return nil
	}

	broadcastQ.QueueBroadcast(&tableEventMessage{
		msg:   raw,
		id:    nid,
		tname: tname,
		key:   key,
		node:  nDB.config.NodeName,
	})
	return nil
}
<commit_msg>Do not invalidate table event messages<commit_after>package networkdb

import (
	"errors"
	"time"

	"github.com/hashicorp/memberlist"
	"github.com/hashicorp/serf/serf"
)

const broadcastTimeout = 5 * time.Second

type networkEventMessage struct {
	id   string
	node string
	msg  []byte
}

func (m *networkEventMessage) Invalidates(other memberlist.Broadcast) bool {
	otherm := other.(*networkEventMessage)
	return m.id == otherm.id && m.node == otherm.node
}

func (m *networkEventMessage) Message() []byte {
	return m.msg
}

func (m *networkEventMessage) Finished() {
}

func (nDB *NetworkDB) sendNetworkEvent(nid string, event NetworkEvent_Type, ltime serf.LamportTime) error {
	nEvent := NetworkEvent{
		Type:      event,
		LTime:     ltime,
		NodeName:  nDB.config.NodeName,
		NetworkID: nid,
	}

	raw, err := encodeMessage(MessageTypeNetworkEvent, &nEvent)
	if err != nil {
		return err
	}

	nDB.networkBroadcasts.QueueBroadcast(&networkEventMessage{
		msg:  raw,
		id:   nid,
		node: nDB.config.NodeName,
	})
	return nil
}

type nodeEventMessage struct {
	msg    []byte
	notify chan<- struct{}
}

func (m *nodeEventMessage) Invalidates(other memberlist.Broadcast) bool {
	return false
}

func (m *nodeEventMessage) Message() []byte {
	return m.msg
}

func (m *nodeEventMessage) Finished() {
	if m.notify != nil {
		close(m.notify)
	}
}

func (nDB *NetworkDB) sendNodeEvent(event NodeEvent_Type) error {
	nEvent := NodeEvent{
		Type:     event,
		LTime:    nDB.networkClock.Increment(),
		NodeName: nDB.config.NodeName,
	}

	raw, err := encodeMessage(MessageTypeNodeEvent, &nEvent)
	if err != nil {
		return err
	}

	notifyCh := make(chan struct{})
	nDB.nodeBroadcasts.QueueBroadcast(&nodeEventMessage{
		msg:    raw,
		notify: notifyCh,
	})

	// Wait for the broadcast
	select {
	case <-notifyCh:
	case <-time.After(broadcastTimeout):
		return errors.New("timed out broadcasting node event")
	}

	return nil
}

type tableEventMessage struct {
	id    string
	tname string
	key   string
	msg   []byte
	node  string
}

func (m *tableEventMessage) Invalidates(other memberlist.Broadcast) bool {
	return false
}

func (m *tableEventMessage) Message() []byte {
	return m.msg
}

func (m *tableEventMessage) Finished() {
}

func (nDB *NetworkDB) sendTableEvent(event TableEvent_Type, nid string, tname string, key string, entry *entry) error {
	tEvent := TableEvent{
		Type:      event,
		LTime:     entry.ltime,
		NodeName:  nDB.config.NodeName,
		NetworkID: nid,
		TableName: tname,
		Key:       key,
		Value:     entry.value,
	}

	raw, err := encodeMessage(MessageTypeTableEvent, &tEvent)
	if err != nil {
		return err
	}

	var broadcastQ *memberlist.TransmitLimitedQueue
	nDB.RLock()
	thisNodeNetworks, ok := nDB.networks[nDB.config.NodeName]
	if ok {
		// The network may have been removed
		network, networkOk := thisNodeNetworks[nid]
		if !networkOk {
			nDB.RUnlock()
			return nil
		}

		broadcastQ = network.tableBroadcasts
	}
	nDB.RUnlock()

	// The network may have been removed
	if broadcastQ == nil {
		return nil
	}

	broadcastQ.QueueBroadcast(&tableEventMessage{
		msg:   raw,
		id:    nid,
		tname: tname,
		key:   key,
		node:  nDB.config.NodeName,
	})
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filesystem

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/config"
	"github.com/andreaskoch/allmark2/common/logger"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/common/util/fsutil"
	"github.com/andreaskoch/allmark2/dataaccess"
	"github.com/andreaskoch/go-fswatch"
	"io/ioutil"
	"path/filepath"
	"strings"
)

var numberOfWatchers int

type Repository struct {
	logger    logger.Logger
	hash      string
	directory string
	watcher   *watcherFactory

	updateHub *UpdateHub

	newItem     chan *dataaccess.RepositoryEvent // new items which are discovered after the first index has been built
	changedItem chan *dataaccess.RepositoryEvent // items with changed content
	movedItem   chan *dataaccess.RepositoryEvent // items which moved
}

func NewRepository(logger logger.Logger, directory string) (*Repository, error) {

	// check if path exists
	if !fsutil.PathExists(directory) {
		return nil, fmt.Errorf("The path %q does not exist.", directory)
	}

	// check if the supplied path is a file
	if isDirectory, _ := fsutil.IsDirectory(directory); !isDirectory {
		directory = filepath.Dir(directory)
	}

	// abort if the supplied path is a reserved directory
	if isReservedDirectory(directory) {
		return nil, fmt.Errorf("The path %q is using a reserved name and cannot be a root.", directory)
	}

	// hash provider: use the directory name for the hash (for now)
	directoryName := strings.ToLower(filepath.Base(directory))
	hash, err := getStringHash(directoryName)
	if err != nil {
		return nil, fmt.Errorf("Cannot create a hash for the repository with the name %q. Error: %s", directoryName, err)
	}

	// enable debug mode
	debugMessages := fswatch.EnableDebug()
	go func() {
		for message := range debugMessages {
			logger.Debug("fs-watch: %s", message)
		}
	}()

	return &Repository{
		logger:    logger,
		directory: directory,
		hash:      hash,
		watcher:   newWatcherFactory(logger),

		updateHub: newUpdateHub(logger),

		newItem:     make(chan *dataaccess.RepositoryEvent, 1),
		changedItem: make(chan *dataaccess.RepositoryEvent, 1),
		movedItem:   make(chan *dataaccess.RepositoryEvent, 1),
	}, nil
}

func (repository *Repository) InitialItems() chan *dataaccess.RepositoryEvent {

	// open the channel
	startupItem := make(chan *dataaccess.RepositoryEvent, 1)

	go func() {

		// repository directory item
		repository.discoverItems(repository.Path(), startupItem)

		// close the channel. All items have been indexed
		close(startupItem)
	}()

	return startupItem
}

func (repository *Repository) UpdateHub() *UpdateHub {
	return repository.updateHub
}

func (repository *Repository) NewItems() chan *dataaccess.RepositoryEvent {
	return repository.newItem
}

func (repository *Repository) ChangedItems() chan *dataaccess.RepositoryEvent {
	return repository.changedItem
}

func (repository *Repository) MovedItems() chan *dataaccess.RepositoryEvent {
	return repository.movedItem
}

func (repository *Repository) Id() string {
	return repository.hash
}

func (repository *Repository) Path() string {
	return repository.directory
}

// Create a new Item for the specified path.
func (repository *Repository) discoverItems(itemPath string, targetChannel chan *dataaccess.RepositoryEvent) {

	// abort if path does not exist
	if !fsutil.PathExists(itemPath) {
		targetChannel <- dataaccess.NewEvent(nil, fmt.Errorf("The path %q does not exist.", itemPath))
		return
	}

	// abort if path is reserved
	if isReservedDirectory(itemPath) {
		targetChannel <- dataaccess.NewEvent(nil, fmt.Errorf("The path %q is using a reserved name and cannot be an item.", itemPath))
		return
	}

	// make sure the item directory points to a folder not a file
	itemDirectory := itemPath
	if isDirectory, _ := fsutil.IsDirectory(itemPath); !isDirectory {
		itemDirectory = filepath.Dir(itemPath)
	}

	// create the item
	item, recurse := repository.getItemFromDirectory(repository.Path(), itemDirectory)

	// send the item to the target channel
	targetChannel <- dataaccess.NewEvent(item, nil)

	// recurse for child items
	if recurse {
		childItemDirectories := getChildDirectories(itemDirectory)
		for _, childItemDirectory := range childItemDirectories {
			repository.discoverItems(childItemDirectory, targetChannel)
		}
	}
}

func (repository *Repository) getItemFromDirectory(repositoryPath, itemDirectory string) (item *dataaccess.Item, recurse bool) {

	// physical item from markdown file
	if found, markdownFilePath := findMarkdownFileInDirectory(itemDirectory); found {

		// create an item from the markdown file
		return repository.newItemFromFile(repositoryPath, itemDirectory, markdownFilePath)

	}

	// virtual item
	if directoryContainsItems(itemDirectory, 3) {
		return repository.newVirtualItem(repositoryPath, itemDirectory)
	}

	// file collection item
	return repository.newFileCollectionItem(repositoryPath, itemDirectory)
}

func (repository *Repository) newItemFromFile(repositoryPath, itemDirectory, filePath string) (item *dataaccess.Item, recurse bool) {
	// route
	route, err := route.NewFromItemPath(repositoryPath, filePath)
	if err != nil {
		repository.logger.Error("Cannot create an Item for the path %q. Error: %s", itemDirectory, err)
		return
	}

	// content provider
	checkIntervalInSeconds := 2
	contentProvider := newFileContentProvider(filePath, route, checkIntervalInSeconds)

	// create the file index
	filesDirectory := filepath.Join(itemDirectory, config.FilesDirectoryName)
	files := getFiles(repositoryPath, itemDirectory, filesDirectory)

	// create the item
	item, err = dataaccess.NewItem(route, contentProvider, files)
	if err != nil {
		repository.logger.Error("Cannot create Item %q. Error: %s", route, err)
		return
	}

	// Update-Hub: Sub-Directory Watcher
	repository.updateHub.Attach(*route, "sub-directory-watcher", func() fswatch.Watcher {
		return repository.watcher.SubDirectories(itemDirectory, 2, func(change *fswatch.FolderChange) {

			// remove the parent item since we cannot easily determine which child has gone away
			go func() {
				repository.movedItem <- dataaccess.NewEvent(item, nil)
			}()

			// recreate this item
			repository.discoverItems(itemDirectory, repository.newItem)
		})
	})

	// Update-Hub: File-Directory Watcher
	repository.updateHub.Attach(*route, "file-directory-watcher", func() fswatch.Watcher {
		return repository.watcher.AllFiles(filesDirectory, 2, func(change *fswatch.FolderChange) {

			// update file list
			repository.logger.Debug("Updating the file list for item %q", item.String())
			newFiles := getFiles(repositoryPath, itemDirectory, filesDirectory)
			item.SetFiles(newFiles)

			// update the parent item
			go func() {
				repository.changedItem <- dataaccess.NewEvent(item, nil)
			}()
		})
	})

	// Update-Hub: Markdown-File Watcher
	repository.updateHub.Attach(*route, "markdown-file-watcher", func() fswatch.Watcher {

		modifiedCallback := func() {
			repository.logger.Debug("Item %q changed.", item)
			go func() {
				repository.changedItem <- dataaccess.NewEvent(item, nil)
			}()
		}

		movedCallback := func() {
			repository.logger.Debug("Item %q moved.", item)
			go func() {
				repository.movedItem <- dataaccess.NewEvent(item, nil)
			}()
		}

		return repository.watcher.File(filePath, 2, modifiedCallback, movedCallback)
	})

	return item, true
}

func (repository *Repository) newVirtualItem(repositoryPath, itemDirectory string) (item *dataaccess.Item, recurse bool) {

	title := filepath.Base(itemDirectory)
	content := fmt.Sprintf(`# %s`, title)

	// route
	route, err := route.NewFromItemDirectory(repositoryPath, itemDirectory)
	if err != nil {
		repository.logger.Error("Cannot create an Item for the path %q. Error: %s", itemDirectory, err)
		return
	}

	// content provider
	contentProvider := newTextContentProvider(content, route)

	// create the file index
	filesDirectory := filepath.Join(itemDirectory, config.FilesDirectoryName)
	files := getFiles(repositoryPath, itemDirectory, filesDirectory)

	// create the item
	item, err = dataaccess.NewItem(route, contentProvider, files)
	if err != nil {
		repository.logger.Error("Cannot create Item %q. Error: %s", route, err)
		return
	}

	// Update-Hub: Sub-Directory Watcher
	repository.updateHub.Attach(*route, "sub-directory-watcher", func() fswatch.Watcher {
		return repository.watcher.SubDirectories(itemDirectory, 2, func(change *fswatch.FolderChange) {

			// remove the parent item since we cannot easily determine which child has gone away
			go func() {
				repository.movedItem <- dataaccess.NewEvent(item, nil)
			}()

			// recreate this item
			repository.discoverItems(itemDirectory, repository.newItem)
		})
	})

	// Update-Hub: Type-Change Watcher
	repository.updateHub.Attach(*route, "type-change-watcher", func() fswatch.Watcher {
		return repository.watcher.Directory(itemDirectory, 2, func(change *fswatch.FolderChange) {

			for _, newFile := range change.New() {
				if !isMarkdownFile(newFile) {
					continue
				}

				repository.watcher.Stop(itemDirectory)

				// remove the parent item since we cannot easily determine which child has gone away
				go func() {
					repository.movedItem <- dataaccess.NewEvent(item, nil)
				}()

				// recreate this item
				repository.discoverItems(itemDirectory, repository.newItem)

				break

			}
		})
	})

	return item, true
}

func (repository *Repository) newFileCollectionItem(repositoryPath, itemDirectory string) (item *dataaccess.Item, recurse bool) {

	title := filepath.Base(itemDirectory)
	content := fmt.Sprintf(`# %s`, title)

	// route
	route, err := route.NewFromItemDirectory(repositoryPath, itemDirectory)
	if err != nil {
		repository.logger.Error("Cannot create an Item for the path %q. Error: %s", itemDirectory, err)
		return
	}

	// content provider
	contentProvider := newTextContentProvider(content, route)

	// create the file index
	filesDirectory := itemDirectory
	files := getFiles(repositoryPath, itemDirectory, filesDirectory)

	// create the item
	item, err = dataaccess.NewItem(route, contentProvider, files)
	if err != nil {
		repository.logger.Error("Cannot create Item %q. Error: %s", route, err)
		return
	}

	// Update-Hub: File-Change Watcher
	repository.updateHub.Attach(*route, "file-change-watcher", func() fswatch.Watcher {
		return repository.watcher.AllFiles(itemDirectory, 2, func(change *fswatch.FolderChange) {

			if directoryContainsItems(itemDirectory, 1) {
				// type change

				// remove the parent item since we cannot easily determine which child has gone away
				go func() {
					repository.movedItem <- dataaccess.NewEvent(item, nil)
				}()

				// recreate this item
				repository.discoverItems(itemDirectory, repository.newItem)

			} else {

				// update files
				newFiles := getFiles(repositoryPath, itemDirectory, filesDirectory)
				item.SetFiles(newFiles)

				// update the parent item
				go func() {
					repository.changedItem <- dataaccess.NewEvent(item, nil)
				}()
			}
		})
	})

	return item, false
}

func directoryContainsItems(directory string, maxdepth int) bool {

	directoryEntries, _ := ioutil.ReadDir(directory)
	for _, entry := range directoryEntries {

		childDirectory := filepath.Join(directory, entry.Name())

		if entry.IsDir() {
			if isReservedDirectory(childDirectory) {
				continue
			}

			if maxdepth > 0 {

				// recurse
				if directoryContainsItems(childDirectory, maxdepth-1) {
					return true
				}
			}

			continue
		}

		if isMarkdownFile(childDirectory) {
			return true
		}

		continue
	}

	return false
}
<commit_msg>Added a file directory watcher for virtual items<commit_after>// Copyright 2013 Andreas Koch. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filesystem

import (
	"fmt"
	"github.com/andreaskoch/allmark2/common/config"
	"github.com/andreaskoch/allmark2/common/logger"
	"github.com/andreaskoch/allmark2/common/route"
	"github.com/andreaskoch/allmark2/common/util/fsutil"
	"github.com/andreaskoch/allmark2/dataaccess"
	"github.com/andreaskoch/go-fswatch"
	"io/ioutil"
	"path/filepath"
	"strings"
)

var numberOfWatchers int

type Repository struct {
	logger    logger.Logger
	hash      string
	directory string
	watcher   *watcherFactory

	updateHub *UpdateHub

	newItem     chan *dataaccess.RepositoryEvent // new items which are discovered after the first index has been built
	changedItem chan *dataaccess.RepositoryEvent // items with changed content
	movedItem   chan *dataaccess.RepositoryEvent // items which moved
}

func NewRepository(logger logger.Logger, directory string) (*Repository, error) {

	// check if path exists
	if !fsutil.PathExists(directory) {
		return nil, fmt.Errorf("The path %q does not exist.", directory)
	}

	// check if the supplied path is a file
	if isDirectory, _ := fsutil.IsDirectory(directory); !isDirectory {
		directory = filepath.Dir(directory)
	}

	// abort if the supplied path is a reserved directory
	if isReservedDirectory(directory) {
		return nil, fmt.Errorf("The path %q is using a reserved name and cannot be a root.", directory)
	}

	// hash provider: use the directory name for the hash (for now)
	directoryName := strings.ToLower(filepath.Base(directory))
	hash, err := getStringHash(directoryName)
	if err != nil {
		return nil, fmt.Errorf("Cannot create a hash for the repository with the name %q. Error: %s", directoryName, err)
	}

	// enable debug mode
	debugMessages := fswatch.EnableDebug()
	go func() {
		for message := range debugMessages {
			logger.Debug("fs-watch: %s", message)
		}
	}()

	return &Repository{
		logger:    logger,
		directory: directory,
		hash:      hash,
		watcher:   newWatcherFactory(logger),

		updateHub: newUpdateHub(logger),

		newItem:     make(chan *dataaccess.RepositoryEvent, 1),
		changedItem: make(chan *dataaccess.RepositoryEvent, 1),
		movedItem:   make(chan *dataaccess.RepositoryEvent, 1),
	}, nil
}

func (repository *Repository) InitialItems() chan *dataaccess.RepositoryEvent {

	// open the channel
	startupItem := make(chan *dataaccess.RepositoryEvent, 1)

	go func() {

		// repository directory item
		repository.discoverItems(repository.Path(), startupItem)

		// close the channel. All items have been indexed
		close(startupItem)
	}()

	return startupItem
}

func (repository *Repository) UpdateHub() *UpdateHub {
	return repository.updateHub
}

func (repository *Repository) NewItems() chan *dataaccess.RepositoryEvent {
	return repository.newItem
}

func (repository *Repository) ChangedItems() chan *dataaccess.RepositoryEvent {
	return repository.changedItem
}

func (repository *Repository) MovedItems() chan *dataaccess.RepositoryEvent {
	return repository.movedItem
}

func (repository *Repository) Id() string {
	return repository.hash
}

func (repository *Repository) Path() string {
	return repository.directory
}

// Create a new Item for the specified path.
func (repository *Repository) discoverItems(itemPath string, targetChannel chan *dataaccess.RepositoryEvent) {

	// abort if path does not exist
	if !fsutil.PathExists(itemPath) {
		targetChannel <- dataaccess.NewEvent(nil, fmt.Errorf("The path %q does not exist.", itemPath))
		return
	}

	// abort if path is reserved
	if isReservedDirectory(itemPath) {
		targetChannel <- dataaccess.NewEvent(nil, fmt.Errorf("The path %q is using a reserved name and cannot be an item.", itemPath))
		return
	}

	// make sure the item directory points to a folder not a file
	itemDirectory := itemPath
	if isDirectory, _ := fsutil.IsDirectory(itemPath); !isDirectory {
		itemDirectory = filepath.Dir(itemPath)
	}

	// create the item
	item, recurse := repository.getItemFromDirectory(repository.Path(), itemDirectory)

	// send the item to the target channel
	targetChannel <- dataaccess.NewEvent(item, nil)

	// recurse for child items
	if recurse {
		childItemDirectories := getChildDirectories(itemDirectory)
		for _, childItemDirectory := range childItemDirectories {
			repository.discoverItems(childItemDirectory, targetChannel)
		}
	}
}

func (repository *Repository) getItemFromDirectory(repositoryPath, itemDirectory string) (item *dataaccess.Item, recurse bool) {

	// physical item from markdown file
	if found, markdownFilePath := findMarkdownFileInDirectory(itemDirectory); found {

		// create an item from the markdown file
		return repository.newItemFromFile(repositoryPath, itemDirectory, markdownFilePath)

	}

	// virtual item
	if directoryContainsItems(itemDirectory, 3) {
		return repository.newVirtualItem(repositoryPath, itemDirectory)
	}

	// file collection item
	return repository.newFileCollectionItem(repositoryPath, itemDirectory)
}

func (repository *Repository) newItemFromFile(repositoryPath, itemDirectory, filePath string) (item *dataaccess.Item, recurse bool) {
	// route
	route, err := route.NewFromItemPath(repositoryPath, filePath)
	if err != nil {
		repository.logger.Error("Cannot create an Item for the path %q. Error: %s", itemDirectory, err)
		return
	}

	// content provider
	checkIntervalInSeconds := 2
	contentProvider := newFileContentProvider(filePath, route, checkIntervalInSeconds)

	// create the file index
	filesDirectory := filepath.Join(itemDirectory, config.FilesDirectoryName)
	files := getFiles(repositoryPath, itemDirectory, filesDirectory)

	// create the item
	item, err = dataaccess.NewItem(route, contentProvider, files)
	if err != nil {
		repository.logger.Error("Cannot create Item %q. Error: %s", route, err)
		return
	}

	// Update-Hub: Sub-Directory Watcher
	repository.updateHub.Attach(*route, "sub-directory-watcher", func() fswatch.Watcher {
		return repository.watcher.SubDirectories(itemDirectory, 2, func(change *fswatch.FolderChange) {

			// remove the parent item since we cannot easily determine which child has gone away
			go func() {
				repository.movedItem <- dataaccess.NewEvent(item, nil)
			}()

			// recreate this item
			repository.discoverItems(itemDirectory, repository.newItem)
		})
	})

	// Update-Hub: File-Directory Watcher
	repository.updateHub.Attach(*route, "file-directory-watcher", func() fswatch.Watcher {
		return repository.watcher.AllFiles(filesDirectory, 2, func(change *fswatch.FolderChange) {

			// update file list
			repository.logger.Debug("Updating the file list for item %q", item.String())
			newFiles := getFiles(repositoryPath, itemDirectory, filesDirectory)
			item.SetFiles(newFiles)

			// update the parent item
			go func() {
				repository.changedItem <- dataaccess.NewEvent(item, nil)
			}()
		})
	})

	// Update-Hub: Markdown-File Watcher
	repository.updateHub.Attach(*route, "markdown-file-watcher", func() fswatch.Watcher {

		modifiedCallback := func() {
			repository.logger.Debug("Item %q changed.", item)
			go func() {
				repository.changedItem <- dataaccess.NewEvent(item, nil)
			}()
		}

		movedCallback := func() {
			repository.logger.Debug("Item %q moved.", item)
			go func() {
				repository.movedItem <- dataaccess.NewEvent(item, nil)
			}()
		}

		return repository.watcher.File(filePath, 2, modifiedCallback, movedCallback)
	})

	return item, true
}

func (repository *Repository) newVirtualItem(repositoryPath, itemDirectory string) (item *dataaccess.Item, recurse bool) {

	title := filepath.Base(itemDirectory)
	content := fmt.Sprintf(`# %s`, title)

	// route
	route, err := route.NewFromItemDirectory(repositoryPath, itemDirectory)
	if err != nil {
		repository.logger.Error("Cannot create an Item for the path %q. Error: %s", itemDirectory, err)
		return
	}

	// content provider
	contentProvider := newTextContentProvider(content, route)

	// create the file index
	filesDirectory := filepath.Join(itemDirectory, config.FilesDirectoryName)
	files := getFiles(repositoryPath, itemDirectory, filesDirectory)

	// create the item
	item, err = dataaccess.NewItem(route, contentProvider, files)
	if err != nil {
		repository.logger.Error("Cannot create Item %q. Error: %s", route, err)
		return
	}

	// Update-Hub: Sub-Directory Watcher
	repository.updateHub.Attach(*route, "sub-directory-watcher", func() fswatch.Watcher {
		return repository.watcher.SubDirectories(itemDirectory, 2, func(change *fswatch.FolderChange) {

			// remove the parent item since we cannot easily determine which child has gone away
			go func() {
				repository.movedItem <- dataaccess.NewEvent(item, nil)
			}()

			// recreate this item
			repository.discoverItems(itemDirectory, repository.newItem)
		})
	})

	// Update-Hub: Type-Change Watcher
	repository.updateHub.Attach(*route, "type-change-watcher", func() fswatch.Watcher {
		return repository.watcher.Directory(itemDirectory, 2, func(change *fswatch.FolderChange) {

			for _, newFile := range change.New() {
				if !isMarkdownFile(newFile) {
					continue
				}

				repository.watcher.Stop(itemDirectory)

				// remove the parent item since we cannot easily determine which child has gone away
				go func() {
					repository.movedItem <- dataaccess.NewEvent(item, nil)
				}()

				// recreate this item
				repository.discoverItems(itemDirectory, repository.newItem)

				break

			}
		})
	})

	// Update-Hub: File-Directory Watcher
	repository.updateHub.Attach(*route, "file-directory-watcher", func() fswatch.Watcher {
		return repository.watcher.AllFiles(filesDirectory, 2, func(change *fswatch.FolderChange) {

			// update file list
			repository.logger.Debug("Updating the file list for item %q", item.String())
			newFiles := getFiles(repositoryPath, itemDirectory, filesDirectory)
			item.SetFiles(newFiles)

			// update the parent item
			go func() {
				repository.changedItem <- dataaccess.NewEvent(item, nil)
			}()
		})
	})

	return item, true
}

func (repository *Repository) newFileCollectionItem(repositoryPath, itemDirectory string) (item *dataaccess.Item, recurse bool) {

	title := filepath.Base(itemDirectory)
	content := fmt.Sprintf(`# %s`, title)

	// route
	route, err := route.NewFromItemDirectory(repositoryPath, itemDirectory)
	if err != nil {
		repository.logger.Error("Cannot create an Item for the path %q. Error: %s", itemDirectory, err)
		return
	}

	// content provider
	contentProvider := newTextContentProvider(content, route)

	// create the file index
	filesDirectory := itemDirectory
	files := getFiles(repositoryPath, itemDirectory, filesDirectory)

	// create the item
	item, err = dataaccess.NewItem(route, contentProvider, files)
	if err != nil {
		repository.logger.Error("Cannot create Item %q. Error: %s", route, err)
		return
	}

	// Update-Hub: File-Change Watcher
	repository.updateHub.Attach(*route, "file-change-watcher", func() fswatch.Watcher {
		return repository.watcher.AllFiles(itemDirectory, 2, func(change *fswatch.FolderChange) {

			if directoryContainsItems(itemDirectory, 1) {
				// type change

				// remove the parent item since we cannot easily determine which child has gone away
				go func() {
					repository.movedItem <- dataaccess.NewEvent(item, nil)
				}()

				// recreate this item
				repository.discoverItems(itemDirectory, repository.newItem)

			} else {

				// update files
				newFiles := getFiles(repositoryPath, itemDirectory, filesDirectory)
				item.SetFiles(newFiles)

				// update the parent item
				go func() {
					repository.changedItem <- dataaccess.NewEvent(item, nil)
				}()
			}
		})
	})

	return item, false
}

func directoryContainsItems(directory string, maxdepth int) bool {

	directoryEntries, _ := ioutil.ReadDir(directory)
	for _, entry := range directoryEntries {

		childDirectory := filepath.Join(directory, entry.Name())

		if entry.IsDir() {
			if isReservedDirectory(childDirectory) {
				continue
			}

			if maxdepth > 0 {

				// recurse
				if directoryContainsItems(childDirectory, maxdepth-1) {
					return true
				}
			}

			continue
		}

		if isMarkdownFile(childDirectory) {
			return true
		}

		continue
	}

	return false
}
<|endoftext|>


<commit_before>package actors_test

import (
	"errors"
	"os"
	"path/filepath"

	"github.com/cloudfoundry/cli/cf/actors"
	fakeBits "github.com/cloudfoundry/cli/cf/api/application_bits/fakes"
	"github.com/cloudfoundry/cli/cf/api/resources"
	"github.com/cloudfoundry/cli/cf/app_files/fakes"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/gofileutils/fileutils"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Push Actor", func() {
	var (
		appBitsRepo  *fakeBits.FakeApplicationBitsRepository
		appFiles     *fakes.FakeAppFiles
		zipper       *fakes.FakeZipper
		actor        actors.PushActor
		fixturesDir  string
		appDir       string
		allFiles     []models.AppFileFields
		presentFiles []resources.AppFileResource
	)

	BeforeEach(func() {
		appBitsRepo = &fakeBits.FakeApplicationBitsRepository{}
		appFiles = &fakes.FakeAppFiles{}
		zipper = &fakes.FakeZipper{}
		actor = actors.NewPushActor(appBitsRepo, zipper, appFiles)
		fixturesDir = filepath.Join("..", "..", "fixtures", "applications")
	})

	Describe("GatherFiles", func() {
		BeforeEach(func() {
			allFiles = []models.AppFileFields{
				models.AppFileFields{Path: "example-app/.cfignore"},
				models.AppFileFields{Path: "example-app/app.rb"},
				models.AppFileFields{Path: "example-app/config.ru"},
				models.AppFileFields{Path: "example-app/Gemfile"},
				models.AppFileFields{Path: "example-app/Gemfile.lock"},
				models.AppFileFields{Path: "example-app/ignore-me"},
				models.AppFileFields{Path: "example-app/manifest.yml"},
			}

			presentFiles = []resources.AppFileResource{
				resources.AppFileResource{Path: "example-app/ignore-me"},
			}

			appDir = filepath.Join(fixturesDir, "example-app.zip")
			zipper.UnzipReturns(nil)
			appFiles.AppFilesInDirReturns(allFiles, nil)
			appBitsRepo.GetApplicationFilesReturns(presentFiles, nil)
		})

		AfterEach(func() {
		})

		Context("when the input is a zipfile", func() {
			BeforeEach(func() {
				zipper.IsZipFileReturns(true)
			})

			It("extracts the zip", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					files, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(zipper.UnzipCallCount()).To(Equal(1))
					Expect(err).NotTo(HaveOccurred())
					Expect(files).To(Equal(presentFiles))
				})
			})

		})

		Context("when the input is a directory full of files", func() {
			BeforeEach(func() {
				zipper.IsZipFileReturns(false)
			})

			It("does not try to unzip the directory", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					files, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(zipper.UnzipCallCount()).To(Equal(0))
					Expect(err).NotTo(HaveOccurred())
					Expect(files).To(Equal(presentFiles))
				})
			})
		})

		Context("when errors occur", func() {
			It("returns an error if it cannot unzip the files", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					zipper.IsZipFileReturns(true)
					zipper.UnzipReturns(errors.New("error"))
					_, _, err = actor.GatherFiles(appDir, tmpDir)
					Expect(err).To(HaveOccurred())
				})
			})

			It("returns an error if it cannot walk the files", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					appFiles.AppFilesInDirReturns(nil, errors.New("error"))
					_, _, err = actor.GatherFiles(appDir, tmpDir)
					Expect(err).To(HaveOccurred())
				})
			})

			It("returns an error if we cannot reach the cc", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					appBitsRepo.GetApplicationFilesReturns(nil, errors.New("error"))
					_, _, err = actor.GatherFiles(appDir, tmpDir)
					Expect(err).To(HaveOccurred())
				})
			})
		})

		Context("when using .cfignore", func() {
			BeforeEach(func() {
				appBitsRepo.GetApplicationFilesReturns(nil, nil)
				appDir = filepath.Join(fixturesDir, "exclude-a-default-cfignore")
			})

			It("includes the .cfignore file in the upload directory", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					files, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(err).NotTo(HaveOccurred())

					_, err = os.Stat(filepath.Join(tmpDir, ".cfignore"))
					Expect(os.IsNotExist(err)).To(BeFalse())
					Expect(len(files)).To(Equal(0))
				})
			})
		})
	})

	Describe(".PopulateFileMode()", func() {
		It("returns []resources.AppFileResource with file mode populated", func() {
			files := []resources.AppFileResource{
				resources.AppFileResource{Path: "example-app/.cfignore"},
				resources.AppFileResource{Path: "example-app/app.rb"},
				resources.AppFileResource{Path: "example-app/config.ru"},
			}

			files, err := actor.PopulateFileMode(fixturesDir, files)
			Ω(err).NotTo(HaveOccurred())
			Ω(files[0].Mode).To(Equal("0644"))
			Ω(files[1].Mode).To(Equal("0755"))
			Ω(files[2].Mode).To(Equal("0644"))
		})
	})

	Describe(".UploadApp", func() {
		It("Simply delegates to the UploadApp function on the app bits repo, which is not worth testing", func() {})
	})
})
<commit_msg>fix filemode test for different platforms<commit_after>package actors_test

import (
	"errors"
	"os"
	"path/filepath"
	"runtime"

	"github.com/cloudfoundry/cli/cf/actors"
	fakeBits "github.com/cloudfoundry/cli/cf/api/application_bits/fakes"
	"github.com/cloudfoundry/cli/cf/api/resources"
	"github.com/cloudfoundry/cli/cf/app_files/fakes"
	"github.com/cloudfoundry/cli/cf/models"
	"github.com/cloudfoundry/gofileutils/fileutils"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Push Actor", func() {
	var (
		appBitsRepo  *fakeBits.FakeApplicationBitsRepository
		appFiles     *fakes.FakeAppFiles
		zipper       *fakes.FakeZipper
		actor        actors.PushActor
		fixturesDir  string
		appDir       string
		allFiles     []models.AppFileFields
		presentFiles []resources.AppFileResource
	)

	BeforeEach(func() {
		appBitsRepo = &fakeBits.FakeApplicationBitsRepository{}
		appFiles = &fakes.FakeAppFiles{}
		zipper = &fakes.FakeZipper{}
		actor = actors.NewPushActor(appBitsRepo, zipper, appFiles)
		fixturesDir = filepath.Join("..", "..", "fixtures", "applications")
	})

	Describe("GatherFiles", func() {
		BeforeEach(func() {
			allFiles = []models.AppFileFields{
				models.AppFileFields{Path: "example-app/.cfignore"},
				models.AppFileFields{Path: "example-app/app.rb"},
				models.AppFileFields{Path: "example-app/config.ru"},
				models.AppFileFields{Path: "example-app/Gemfile"},
				models.AppFileFields{Path: "example-app/Gemfile.lock"},
				models.AppFileFields{Path: "example-app/ignore-me"},
				models.AppFileFields{Path: "example-app/manifest.yml"},
			}

			presentFiles = []resources.AppFileResource{
				resources.AppFileResource{Path: "example-app/ignore-me"},
			}

			appDir = filepath.Join(fixturesDir, "example-app.zip")
			zipper.UnzipReturns(nil)
			appFiles.AppFilesInDirReturns(allFiles, nil)
			appBitsRepo.GetApplicationFilesReturns(presentFiles, nil)
		})

		AfterEach(func() {
		})

		Context("when the input is a zipfile", func() {
			BeforeEach(func() {
				zipper.IsZipFileReturns(true)
			})

			It("extracts the zip", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					files, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(zipper.UnzipCallCount()).To(Equal(1))
					Expect(err).NotTo(HaveOccurred())
					Expect(files).To(Equal(presentFiles))
				})
			})

		})

		Context("when the input is a directory full of files", func() {
			BeforeEach(func() {
				zipper.IsZipFileReturns(false)
			})

			It("does not try to unzip the directory", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					files, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(zipper.UnzipCallCount()).To(Equal(0))
					Expect(err).NotTo(HaveOccurred())
					Expect(files).To(Equal(presentFiles))
				})
			})
		})

		Context("when errors occur", func() {
			It("returns an error if it cannot unzip the files", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					zipper.IsZipFileReturns(true)
					zipper.UnzipReturns(errors.New("error"))
					_, _, err = actor.GatherFiles(appDir, tmpDir)
					Expect(err).To(HaveOccurred())
				})
			})

			It("returns an error if it cannot walk the files", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					appFiles.AppFilesInDirReturns(nil, errors.New("error"))
					_, _, err = actor.GatherFiles(appDir, tmpDir)
					Expect(err).To(HaveOccurred())
				})
			})

			It("returns an error if we cannot reach the cc", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					appBitsRepo.GetApplicationFilesReturns(nil, errors.New("error"))
					_, _, err = actor.GatherFiles(appDir, tmpDir)
					Expect(err).To(HaveOccurred())
				})
			})
		})

		Context("when using .cfignore", func() {
			BeforeEach(func() {
				appBitsRepo.GetApplicationFilesReturns(nil, nil)
				appDir = filepath.Join(fixturesDir, "exclude-a-default-cfignore")
			})

			It("includes the .cfignore file in the upload directory", func() {
				fileutils.TempDir("gather-files", func(tmpDir string, err error) {
					files, _, err := actor.GatherFiles(appDir, tmpDir)
					Expect(err).NotTo(HaveOccurred())

					_, err = os.Stat(filepath.Join(tmpDir, ".cfignore"))
					Expect(os.IsNotExist(err)).To(BeFalse())
					Expect(len(files)).To(Equal(0))
				})
			})
		})
	})

	Describe(".PopulateFileMode()", func() {
		It("returns []resources.AppFileResource with file mode populated", func() {
			files := []resources.AppFileResource{
				resources.AppFileResource{Path: "example-app/.cfignore"},
				resources.AppFileResource{Path: "example-app/app.rb"},
				resources.AppFileResource{Path: "example-app/config.ru"},
			}

			files, err := actor.PopulateFileMode(fixturesDir, files)
			Ω(err).NotTo(HaveOccurred())
			if runtime.GOOS == "windows" { //windows filemode is different
				Ω(files[0].Mode).ToNot(Equal(""))
				Ω(files[1].Mode).ToNot(Equal(""))
				Ω(files[2].Mode).ToNot(Equal(""))
			} else if runtime.GOARCH == "386" { //32bit filemode is different
				Ω(files[0].Mode).ToNot(Equal(""))
				Ω(files[1].Mode).ToNot(Equal(""))
				Ω(files[2].Mode).ToNot(Equal(""))
			} else { //linux and osx 64bit
				Ω(files[0].Mode).To(Equal("0644"))
				Ω(files[1].Mode).To(Equal("0755"))
				Ω(files[2].Mode).To(Equal("0644"))
			}
		})
	})

	Describe(".UploadApp", func() {
		It("Simply delegates to the UploadApp function on the app bits repo, which is not worth testing", func() {})
	})
})
<|endoftext|>