<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagman/processutil"
	"github.com/APTrust/bagman/ingesthelper"
	"github.com/bitly/go-nsq"
	"path/filepath"
	"time"
)

type Channels struct {
	StorageChannel chan *ingesthelper.IngestHelper
	CleanUpChannel chan *ingesthelper.IngestHelper
	ResultsChannel chan *ingesthelper.IngestHelper
}

// Global vars.
var procUtil *processutil.ProcessUtil
var channels *Channels

// apt_store stores bags that have been unpacked and validated
// by apt_prepare. Each item/message follows this flow:
//
// 1. Storage channel: copies files to S3 permanent storage.
// 2. Results channel: tells the queue whether processing
//    succeeded, and if not, whether the item should be requeued.
//    Also logs results to json and message logs.
// 3. Cleanup channel: cleans up the files after processing
//    completes.
//
// If a failure occurs anywhere in the first step,
// processing goes directly to the Results Channel, which
// records the error and the disposition (retry/give up).
//
// As long as the message from nsq contains valid JSON,
// steps 2 and 3 ALWAYS run.
func main() {
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	procUtil = processutil.NewProcessUtil(requestedConfig)

	procUtil.MessageLog.Info("apt_store started")
	err := procUtil.Config.EnsureFluctusConfig()
	if err != nil {
		procUtil.MessageLog.Fatalf("Required Fluctus config vars are missing: %v", err)
	}

	initChannels()
	initGoRoutines()

	nsqConfig := nsq.NewConfig()
	nsqConfig.Set("max_in_flight", 20)
	nsqConfig.Set("heartbeat_interval", "10s")
	nsqConfig.Set("max_attempts", uint16(procUtil.Config.MaxStoreAttempts))
	nsqConfig.Set("read_timeout", "60s")
	nsqConfig.Set("write_timeout", "10s")
	nsqConfig.Set("msg_timeout", "180m")
	consumer, err := nsq.NewConsumer(procUtil.Config.StoreTopic,
		procUtil.Config.StoreChannel, nsqConfig)
	if err != nil {
		procUtil.MessageLog.Fatalf(err.Error())
	}

	handler := &APTStore{}
	consumer.SetHandler(handler)
	consumer.ConnectToNSQLookupd(procUtil.Config.NsqLookupd)

	// This reader blocks until we get an interrupt, so our program does not exit.
	<-consumer.StopChan
}

// Set up the channels. It's essential that that the fetchChannel
// be limited to a relatively low number. If we are downloading
// 1GB tar files, we need space to store the tar file AND the
// untarred version. That's about 2 x 1GB. We do not want to pull
// down 1000 files at once, or we'll run out of disk space!
// If config sets fetchers to 10, we can pull down 10 files at a
// time. The fetch queue could hold 10 * 4 = 40 items, so we'd
// have max 40 tar files + untarred directories on disk at once.
// The number of workers should be close to the number of CPU
// cores.
func initChannels() {
	workerBufferSize := procUtil.Config.StoreWorkers * 10

	channels = &Channels{}
	channels.StorageChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.CleanUpChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.ResultsChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
}

// Set up our go routines. We do NOT want one go routine per
// S3 file. If we do that, the system will run out of file handles,
// as we'll have tens of thousands of open connections to S3
// trying to write data into tens of thousands of local files.
func initGoRoutines() {
	for i := 0; i < procUtil.Config.StoreWorkers; i++ {
		go saveToStorage()
		go logResult()
		go doCleanUp()
	}
}

type APTStore struct {
}

// MessageHandler handles messages from the queue, putting each
// item into the pipleline.
func (*APTStore) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var result bagman.ProcessResult
	err := json.Unmarshal(message.Body, &result)
	if err != nil {
		procUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}

	if result.BagReadResult == nil {
		procUtil.MessageLog.Error("Result.BagReadResult is nil")
		message.Finish()
		return fmt.Errorf("Result.BagReadResult is nil")
	}

	// Special case for very large bags: the bag is in process under
	// the same ID. NSQ thinks it timed out and has re-sent it. In this
	// case, return nil so NSQ knows we're OK, but don't finish the message.
	// The original process will call Finish() on the message when it's
	// done. If we call Finish() here, NSQ will throw a "not-in-flight"
	// error when the processor calls Finish() on the original message later.
	if procUtil.MessageIdFor(result.S3File.BagName()) != "" {
		procUtil.MessageLog.Info("Skipping bag %s: already in progress",
			result.S3File.Key.Key)
		message.Finish()
		return nil
	}

	if allFilesExist(result.TarResult.OutputDir, result.TarResult.GenericFiles) == false {
		procUtil.MessageLog.Error("Cannot process %s because of missing file(s)",
			result.S3File.BagName())
		message.Finish()
		return fmt.Errorf("At least one data file does not exist")
	}

	// Don't start working on a message that we're already working on.
	// Note that the key we include in the syncMap includes multipart
	// bag endings, so we can be working on ncsu.edu/obj.b1of2.tar and
	// ncsu.edu/obj.b2of2.tar at the same time. This is what we want.
	mapErr := procUtil.RegisterItem(result.S3File.BagName(), message.ID)
	if mapErr != nil {
		procUtil.MessageLog.Info("Marking %s as complete because the file is already "+
			"being processed under another message id.\n", result.S3File.Key.Key)
		message.Finish()
		return nil
	}

	// Create the result struct and pass it down the pipeline
	helper := ingesthelper.NewIngestHelper(procUtil, message, result.S3File)
	helper.Result = &result
	helper.Result.NsqMessage = message
	channels.StorageChannel <- helper
	procUtil.MessageLog.Debug("Put %s into storage queue", result.S3File.Key.Key)
	return nil
}

// -- Step 1 of 3 --
// This runs as a go routine to save generic files to the permanent
// S3 storage bucket. Unfortunately, there is no concept of transaction
// here. Ideally, either all GenericFiles make to S3 or none do. In
// cases where we're updating an existing bag (i.e. user uploaded a new
// version of it), we may wind up in a state where half of the new files
// make it successfully to S3 and half do not. That would leave us with
// an inconsistent bag, containing half new files and half old files.
// In addition, the failure to copy all files to S3 would result in no
// metadata going to Fedora. So Fedora would not show that any of the
// generic files were overwritten, even though some were. We should alert
// an admin in these cases. The JSON log will have full information about
// the state of all of the files.
func saveToStorage() {
	for helper := range channels.StorageChannel {
		// Touch before and after sending generic files,
		// since that process can take a long time for large bags.
		helper.Result.NsqMessage.Touch()
		helper.UpdateFluctusStatus(bagman.StageStore, bagman.StatusStarted)
		err := helper.SaveGenericFiles()
		helper.Result.NsqMessage.Touch()
		if err != nil {
			channels.ResultsChannel <- helper
			continue
		}
		// If there were no errors, put this into the metadata
		// queue, so we can record the events in Fluctus.
		if helper.Result.ErrorMessage == "" {
			helper.UpdateFluctusStatus(bagman.StageStore, bagman.StatusPending)
			SendToMetadataQueue(helper)
		}

		// Pass problem cases off to the trouble queue
		if helper.IncompleteCopyToS3() || helper.FailedAndNoMoreRetries() {
			SendToTroubleQueue(helper)
		}

		// Record the results.
		channels.ResultsChannel <- helper
	}
}

// -- Step 2 of 3 --
// This prints to the log the result of the program's attempt to fetch,
// untar, unbag and verify an individual S3 tar file. It logs state info
// about this bag to a json file on the local file system. Also logs
// a text message to the local bag_processor.log file and sends info
// to Fluctus saying whether the bag succeeded or failed.
// THIS STEP ALWAYS RUNS, EVEN IF PRIOR STEPS FAILED.
func logResult() {
	for helper := range channels.ResultsChannel {
		helper.LogResult()
		channels.CleanUpChannel <- helper
	}
}

// -- Step 3 of 3 --
// This runs as a go routine to remove the files we downloaded
// and untarred.
// THIS STEP ALWAYS RUNS, EVEN IF PRIOR STEPS FAILED.
func doCleanUp() {
	for helper := range channels.CleanUpChannel {
		result := helper.Result
		result.NsqMessage.Touch()
		procUtil.MessageLog.Debug("Cleaning up %s", result.S3File.Key.Key)
		if (result.S3File.Key.Key != "" && result.FetchResult != nil &&
			result.FetchResult.LocalTarFile != "") {
			// Clean up any files we downloaded and unpacked
			errors := helper.DeleteLocalFiles()
			if errors != nil && len(errors) > 0 {
				procUtil.MessageLog.Warning("Errors cleaning up %s",
					result.FetchResult.LocalTarFile)
				for _, e := range errors {
					procUtil.MessageLog.Error(e.Error())
				}
			}
		}

		// Build and send message back to NSQ, indicating whether
		// processing succeeded.
		if result.ErrorMessage != "" && result.Retry == true {
			procUtil.MessageLog.Info("Requeueing %s", result.S3File.Key.Key)
			result.NsqMessage.Requeue(5 * time.Minute)
		} else {
			result.NsqMessage.Finish()
		}

		// We're done processing this, so remove it from the map.
		// If it comes in again, we'll reprocess it again.
		procUtil.UnregisterItem(result.S3File.BagName())
	}
}

// Puts an item into the queue for Fluctus/Fedora metadata processing.
func SendToMetadataQueue(helper *ingesthelper.IngestHelper) {
	err := bagman.Enqueue(helper.ProcUtil.Config.NsqdHttpAddress,
		helper.ProcUtil.Config.MetadataTopic, helper.Result)
	if err != nil {
		errMsg := fmt.Sprintf("Error adding '%s' to metadata queue: %v ",
			helper.Result.S3File.Key.Key, err)
		helper.ProcUtil.MessageLog.Error(errMsg)
		helper.Result.ErrorMessage += errMsg
	} else {
		helper.ProcUtil.MessageLog.Debug("Sent '%s' to metadata queue",
			helper.Result.S3File.Key.Key)
	}
}

// Puts an item into the trouble queue.
func SendToTroubleQueue(helper *ingesthelper.IngestHelper) {
	err := bagman.Enqueue(helper.ProcUtil.Config.NsqdHttpAddress,
		helper.ProcUtil.Config.TroubleTopic, helper.Result)
	if err != nil {
		helper.ProcUtil.MessageLog.Error("Could not send '%s' to trouble queue: %v\n",
			helper.Result.S3File.Key.Key, err)
	} else {
		reason := "Processing failed and we reached the maximum number of retries."
		if helper.IncompleteCopyToS3() {
			reason = "Some files could not be copied to S3."
		}
		helper.Result.ErrorMessage += fmt.Sprintf("%s This item has been queued for administrative review.",
			reason)
		helper.ProcUtil.MessageLog.Warning("Sent '%s' to trouble queue: %s",
			helper.Result.S3File.Key.Key, reason)
	}
}

func allFilesExist(rootDir string, genericFiles []*bagman.GenericFile) (bool) {
	for _, gf := range genericFiles {
		absPath := filepath.Join(rootDir, gf.Path)
		if bagman.FileExists(absPath) == false {
			procUtil.MessageLog.Error("File %s does not exist", absPath)
			return false
		}
	}
	return true
}
<commit_msg>Skip filecheck<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/APTrust/bagman"
	"github.com/APTrust/bagman/processutil"
	"github.com/APTrust/bagman/ingesthelper"
	"github.com/bitly/go-nsq"
	"path/filepath"
	"time"
)

type Channels struct {
	StorageChannel chan *ingesthelper.IngestHelper
	CleanUpChannel chan *ingesthelper.IngestHelper
	ResultsChannel chan *ingesthelper.IngestHelper
}

// Global vars.
var procUtil *processutil.ProcessUtil
var channels *Channels

// apt_store stores bags that have been unpacked and validated
// by apt_prepare. Each item/message follows this flow:
//
// 1. Storage channel: copies files to S3 permanent storage.
// 2. Results channel: tells the queue whether processing
//    succeeded, and if not, whether the item should be requeued.
//    Also logs results to json and message logs.
// 3. Cleanup channel: cleans up the files after processing
//    completes.
//
// If a failure occurs anywhere in the first step,
// processing goes directly to the Results Channel, which
// records the error and the disposition (retry/give up).
//
// As long as the message from nsq contains valid JSON,
// steps 2 and 3 ALWAYS run.
func main() {
	requestedConfig := flag.String("config", "", "configuration to run")
	flag.Parse()
	procUtil = processutil.NewProcessUtil(requestedConfig)

	procUtil.MessageLog.Info("apt_store started")
	err := procUtil.Config.EnsureFluctusConfig()
	if err != nil {
		procUtil.MessageLog.Fatalf("Required Fluctus config vars are missing: %v", err)
	}

	initChannels()
	initGoRoutines()

	nsqConfig := nsq.NewConfig()
	nsqConfig.Set("max_in_flight", 20)
	nsqConfig.Set("heartbeat_interval", "10s")
	nsqConfig.Set("max_attempts", uint16(procUtil.Config.MaxStoreAttempts))
	nsqConfig.Set("read_timeout", "60s")
	nsqConfig.Set("write_timeout", "10s")
	nsqConfig.Set("msg_timeout", "180m")
	consumer, err := nsq.NewConsumer(procUtil.Config.StoreTopic,
		procUtil.Config.StoreChannel, nsqConfig)
	if err != nil {
		procUtil.MessageLog.Fatalf(err.Error())
	}

	handler := &APTStore{}
	consumer.SetHandler(handler)
	consumer.ConnectToNSQLookupd(procUtil.Config.NsqLookupd)

	// This reader blocks until we get an interrupt, so our program does not exit.
	<-consumer.StopChan
}

// Set up the channels. It's essential that that the fetchChannel
// be limited to a relatively low number. If we are downloading
// 1GB tar files, we need space to store the tar file AND the
// untarred version. That's about 2 x 1GB. We do not want to pull
// down 1000 files at once, or we'll run out of disk space!
// If config sets fetchers to 10, we can pull down 10 files at a
// time. The fetch queue could hold 10 * 4 = 40 items, so we'd
// have max 40 tar files + untarred directories on disk at once.
// The number of workers should be close to the number of CPU
// cores.
func initChannels() {
	workerBufferSize := procUtil.Config.StoreWorkers * 10

	channels = &Channels{}
	channels.StorageChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.CleanUpChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
	channels.ResultsChannel = make(chan *ingesthelper.IngestHelper, workerBufferSize)
}

// Set up our go routines. We do NOT want one go routine per
// S3 file. If we do that, the system will run out of file handles,
// as we'll have tens of thousands of open connections to S3
// trying to write data into tens of thousands of local files.
func initGoRoutines() {
	for i := 0; i < procUtil.Config.StoreWorkers; i++ {
		go saveToStorage()
		go logResult()
		go doCleanUp()
	}
}

type APTStore struct {
}

// MessageHandler handles messages from the queue, putting each
// item into the pipleline.
func (*APTStore) HandleMessage(message *nsq.Message) error {
	message.DisableAutoResponse()
	var result bagman.ProcessResult
	err := json.Unmarshal(message.Body, &result)
	if err != nil {
		procUtil.MessageLog.Error("Could not unmarshal JSON data from nsq:",
			string(message.Body))
		message.Finish()
		return fmt.Errorf("Could not unmarshal JSON data from nsq")
	}

	if result.BagReadResult == nil {
		procUtil.MessageLog.Error("Result.BagReadResult is nil")
		message.Finish()
		return fmt.Errorf("Result.BagReadResult is nil")
	}

	// Special case for very large bags: the bag is in process under
	// the same ID. NSQ thinks it timed out and has re-sent it. In this
	// case, return nil so NSQ knows we're OK, but don't finish the message.
	// The original process will call Finish() on the message when it's
	// done. If we call Finish() here, NSQ will throw a "not-in-flight"
	// error when the processor calls Finish() on the original message later.
	if procUtil.MessageIdFor(result.S3File.BagName()) != "" {
		procUtil.MessageLog.Info("Skipping bag %s: already in progress",
			result.S3File.Key.Key)
		message.Finish()
		return nil
	}

	// NOTE: This is commented out for now, so we can see if it is necessary.
	// It eats resources when bags are large (10,000+ files), and the validate
	// step in apt_prepare should ensure that all files are present.
	//
	// if allFilesExist(result.TarResult.OutputDir, result.TarResult.GenericFiles) == false {
	// 	procUtil.MessageLog.Error("Cannot process %s because of missing file(s)",
	// 		result.S3File.BagName())
	// 	message.Finish()
	// 	return fmt.Errorf("At least one data file does not exist")
	// }

	// Don't start working on a message that we're already working on.
	// Note that the key we include in the syncMap includes multipart
	// bag endings, so we can be working on ncsu.edu/obj.b1of2.tar and
	// ncsu.edu/obj.b2of2.tar at the same time. This is what we want.
	mapErr := procUtil.RegisterItem(result.S3File.BagName(), message.ID)
	if mapErr != nil {
		procUtil.MessageLog.Info("Marking %s as complete because the file is already "+
			"being processed under another message id.\n", result.S3File.Key.Key)
		message.Finish()
		return nil
	}

	// Create the result struct and pass it down the pipeline
	helper := ingesthelper.NewIngestHelper(procUtil, message, result.S3File)
	helper.Result = &result
	helper.Result.NsqMessage = message
	channels.StorageChannel <- helper
	procUtil.MessageLog.Debug("Put %s into storage queue", result.S3File.Key.Key)
	return nil
}

// -- Step 1 of 3 --
// This runs as a go routine to save generic files to the permanent
// S3 storage bucket. Unfortunately, there is no concept of transaction
// here. Ideally, either all GenericFiles make to S3 or none do. In
// cases where we're updating an existing bag (i.e. user uploaded a new
// version of it), we may wind up in a state where half of the new files
// make it successfully to S3 and half do not. That would leave us with
// an inconsistent bag, containing half new files and half old files.
// In addition, the failure to copy all files to S3 would result in no
// metadata going to Fedora. So Fedora would not show that any of the
// generic files were overwritten, even though some were. We should alert
// an admin in these cases. The JSON log will have full information about
// the state of all of the files.
func saveToStorage() {
	for helper := range channels.StorageChannel {
		// Touch before and after sending generic files,
		// since that process can take a long time for large bags.
		helper.Result.NsqMessage.Touch()
		helper.UpdateFluctusStatus(bagman.StageStore, bagman.StatusStarted)
		err := helper.SaveGenericFiles()
		helper.Result.NsqMessage.Touch()
		if err != nil {
			channels.ResultsChannel <- helper
			continue
		}
		// If there were no errors, put this into the metadata
		// queue, so we can record the events in Fluctus.
		if helper.Result.ErrorMessage == "" {
			helper.UpdateFluctusStatus(bagman.StageStore, bagman.StatusPending)
			SendToMetadataQueue(helper)
		}

		// Pass problem cases off to the trouble queue
		if helper.IncompleteCopyToS3() || helper.FailedAndNoMoreRetries() {
			SendToTroubleQueue(helper)
		}

		// Record the results.
		channels.ResultsChannel <- helper
	}
}

// -- Step 2 of 3 --
// This prints to the log the result of the program's attempt to fetch,
// untar, unbag and verify an individual S3 tar file. It logs state info
// about this bag to a json file on the local file system. Also logs
// a text message to the local bag_processor.log file and sends info
// to Fluctus saying whether the bag succeeded or failed.
// THIS STEP ALWAYS RUNS, EVEN IF PRIOR STEPS FAILED.
func logResult() {
	for helper := range channels.ResultsChannel {
		helper.LogResult()
		channels.CleanUpChannel <- helper
	}
}

// -- Step 3 of 3 --
// This runs as a go routine to remove the files we downloaded
// and untarred.
// THIS STEP ALWAYS RUNS, EVEN IF PRIOR STEPS FAILED.
func doCleanUp() {
	for helper := range channels.CleanUpChannel {
		result := helper.Result
		result.NsqMessage.Touch()
		procUtil.MessageLog.Debug("Cleaning up %s", result.S3File.Key.Key)
		if (result.S3File.Key.Key != "" && result.FetchResult != nil &&
			result.FetchResult.LocalTarFile != "") {
			// Clean up any files we downloaded and unpacked
			errors := helper.DeleteLocalFiles()
			if errors != nil && len(errors) > 0 {
				procUtil.MessageLog.Warning("Errors cleaning up %s",
					result.FetchResult.LocalTarFile)
				for _, e := range errors {
					procUtil.MessageLog.Error(e.Error())
				}
			}
		}

		// Build and send message back to NSQ, indicating whether
		// processing succeeded.
		if result.ErrorMessage != "" && result.Retry == true {
			procUtil.MessageLog.Info("Requeueing %s", result.S3File.Key.Key)
			result.NsqMessage.Requeue(5 * time.Minute)
		} else {
			result.NsqMessage.Finish()
		}

		// We're done processing this, so remove it from the map.
		// If it comes in again, we'll reprocess it again.
		procUtil.UnregisterItem(result.S3File.BagName())
	}
}

// Puts an item into the queue for Fluctus/Fedora metadata processing.
func SendToMetadataQueue(helper *ingesthelper.IngestHelper) {
	err := bagman.Enqueue(helper.ProcUtil.Config.NsqdHttpAddress,
		helper.ProcUtil.Config.MetadataTopic, helper.Result)
	if err != nil {
		errMsg := fmt.Sprintf("Error adding '%s' to metadata queue: %v ",
			helper.Result.S3File.Key.Key, err)
		helper.ProcUtil.MessageLog.Error(errMsg)
		helper.Result.ErrorMessage += errMsg
	} else {
		helper.ProcUtil.MessageLog.Debug("Sent '%s' to metadata queue",
			helper.Result.S3File.Key.Key)
	}
}

// Puts an item into the trouble queue.
func SendToTroubleQueue(helper *ingesthelper.IngestHelper) {
	err := bagman.Enqueue(helper.ProcUtil.Config.NsqdHttpAddress,
		helper.ProcUtil.Config.TroubleTopic, helper.Result)
	if err != nil {
		helper.ProcUtil.MessageLog.Error("Could not send '%s' to trouble queue: %v\n",
			helper.Result.S3File.Key.Key, err)
	} else {
		reason := "Processing failed and we reached the maximum number of retries."
		if helper.IncompleteCopyToS3() {
			reason = "Some files could not be copied to S3."
		}
		helper.Result.ErrorMessage += fmt.Sprintf("%s This item has been queued for administrative review.",
			reason)
		helper.ProcUtil.MessageLog.Warning("Sent '%s' to trouble queue: %s",
			helper.Result.S3File.Key.Key, reason)
	}
}

func allFilesExist(rootDir string, genericFiles []*bagman.GenericFile) (bool) {
	for _, gf := range genericFiles {
		absPath := filepath.Join(rootDir, gf.Path)
		if bagman.FileExists(absPath) == false {
			procUtil.MessageLog.Error("File %s does not exist", absPath)
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package blaze

import (
	"bytes"
	"mime/multipart"
	"net/http"
	"net/http/httptest"
	"net/textproto"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/256dpi/fire"
	"github.com/256dpi/fire/coal"
	"github.com/256dpi/fire/heat"
)

// TODO: Test claim keys.

func TestStorageUploadInvalidContentType(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		body := strings.NewReader("Hello World!")
		req := httptest.NewRequest("POST", "/foo", body)

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(0))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusBadRequest, res.Code)
		assert.Equal(t, "", res.Body.String())

		assert.Equal(t, 0, tester.Count(&File{}))
	})
}

func TestStorageUploadRaw(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		service := NewMemory()
		storage := NewStorage(tester.Store, testNotary, service)

		body := strings.NewReader("Hello World!")

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", "application/octet-stream")

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(0))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, res.Code)
		assert.NotEmpty(t, res.Body.String())
		assert.Equal(t, map[string]Blob{
			"1": {
				Type:  "application/octet-stream",
				Bytes: []byte("Hello World!"),
			},
		}, service.Blobs)

		files := *tester.FindAll(&File{}).(*[]*File)
		assert.Len(t, files, 1)
		assert.Equal(t, Uploaded, files[0].State)
		assert.Equal(t, "application/octet-stream", files[0].Type)
		assert.Equal(t, int64(12), files[0].Length)
		assert.Equal(t, Handle{"id": "1"}, files[0].Handle)
	})
}

func TestStorageUploadRawLimit(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		body := strings.NewReader("Hello World!")

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", "application/octet-stream")

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(1))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusRequestEntityTooLarge, res.Code)
		assert.Equal(t, "", res.Body.String())
	})
}

func TestStorageUploadFormFiles(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		service := NewMemory()
		storage := NewStorage(tester.Store, testNotary, service)

		body := &bytes.Buffer{}
		writer := multipart.NewWriter(body)

		part, err := writer.CreateFormFile("file1", "data1.bin")
		assert.NoError(t, err)

		_, err = part.Write([]byte("Hello World 1!"))
		assert.NoError(t, err)

		part, err = writer.CreateFormFile("file2", "data2.bin")
		assert.NoError(t, err)

		_, err = part.Write([]byte("Hello World 2!"))
		assert.NoError(t, err)

		err = writer.Close()
		assert.NoError(t, err)

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", writer.FormDataContentType())

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(0))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, res.Code)
		assert.NotEmpty(t, res.Body.String())

		assert.Equal(t, map[string]Blob{
			"1": {
				Type:  "application/octet-stream",
				Bytes: []byte("Hello World 1!"),
			},
			"2": {
				Type:  "application/octet-stream",
				Bytes: []byte("Hello World 2!"),
			},
		}, service.Blobs)

		files := *tester.FindAll(&File{}).(*[]*File)
		assert.Len(t, files, 2)
		assert.Equal(t, Uploaded, files[0].State)
		assert.Equal(t, "application/octet-stream", files[0].Type)
		assert.Equal(t, int64(14), files[0].Length)
		assert.Equal(t, Handle{"id": "1"}, files[0].Handle)
		assert.Equal(t, Uploaded, files[1].State)
		assert.Equal(t, "application/octet-stream", files[1].Type)
		assert.Equal(t, int64(14), files[1].Length)
		assert.Equal(t, Handle{"id": "2"}, files[1].Handle)
	})
}

func TestStorageUploadFormFilesLimit(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		body := &bytes.Buffer{}
		writer := multipart.NewWriter(body)

		part, err := writer.CreateFormFile("file1", "data1.bin")
		assert.NoError(t, err)

		_, err = part.Write([]byte("Hello World 1!"))
		assert.NoError(t, err)

		err = writer.Close()
		assert.NoError(t, err)

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", writer.FormDataContentType())

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(1))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusRequestEntityTooLarge, res.Code)
		assert.Equal(t, "", res.Body.String())
	})
}

func TestStorageUploadMultipart(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		service := NewMemory()
		storage := NewStorage(tester.Store, testNotary, service)

		body := &bytes.Buffer{}
		writer := multipart.NewWriter(body)

		part, err := writer.CreatePart(textproto.MIMEHeader{
			"Content-Disposition": []string{`form-data; name="file1"; filename="style.css"`},
			"Content-Type":        []string{"text/css"},
		})

		_, err = part.Write([]byte("h1 { color: red; }"))
		assert.NoError(t, err)

		part, err = writer.CreatePart(textproto.MIMEHeader{
			"Content-Disposition": []string{`form-data; name="file2"; filename="script.js"`},
			"Content-Type":        []string{"text/javascript"},
		})

		_, err = part.Write([]byte("console.log('Hello World!);"))
		assert.NoError(t, err)

		err = writer.Close()
		assert.NoError(t, err)

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", writer.FormDataContentType())

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(0))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, res.Code)
		assert.NotEmpty(t, res.Body.String())

		assert.Equal(t, map[string]Blob{
			"1": {
				Type:  "text/css",
				Bytes: []byte("h1 { color: red; }"),
			},
			"2": {
				Type:  "text/javascript",
				Bytes: []byte("console.log('Hello World!);"),
			},
		}, service.Blobs)
	})
}

func TestStorageUploadMultipartLimit(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		body := &bytes.Buffer{}
		writer := multipart.NewWriter(body)

		part, err := writer.CreatePart(textproto.MIMEHeader{
			"Content-Disposition": []string{`form-data; name="file1"; filename="style.css"`},
			"Content-Type":        []string{"text/css"},
		})

		_, err = part.Write([]byte("console.log('Hello World!);"))
		assert.NoError(t, err)

		err = writer.Close()
		assert.NoError(t, err)

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", writer.FormDataContentType())

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(1))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusRequestEntityTooLarge, res.Code)
		assert.Equal(t, "", res.Body.String())
	})
}

func TestStorageValidator(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		validator := storage.Validator()

		/* missing */

		model := &testModel{}
		err := tester.RunCallback(&fire.Context{
			Operation: fire.Create,
			Model:     model,
			Controller: &fire.Controller{
				Model: coal.Init(&testModel{}),
			},
		}, validator)
		assert.Error(t, err)
		assert.Equal(t, "required-file: missing claim key", err.Error())

		/* required */

		file1 := tester.Save(&File{
			State: Uploaded,
		}).(*File)

		claimKey1, err := storage.notary.Issue(&ClaimKey{
			File: file1.ID(),
		})
		assert.NoError(t, err)
		assert.NotEmpty(t, claimKey1)

		model = &testModel{
			RequiredFile: Link{
				ClaimKey: claimKey1,
			},
		}
		err = tester.RunCallback(&fire.Context{
			Operation: fire.Create,
			Model:     model,
			Controller: &fire.Controller{
				Model: coal.Init(&testModel{}),
			},
		}, validator)
		assert.NoError(t, err)
		assert.Equal(t, file1.ID(), *model.RequiredFile.File)

		file1 = tester.Fetch(&File{}, file1.ID()).(*File)
		assert.Equal(t, Claimed, file1.State)
	})
}

func TestStorageDecorator(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		decorator := storage.Decorator()

		file1 := coal.New()
		file2 := coal.New()

		model := &testModel{
			RequiredFile: Link{
				File: coal.P(file1),
			},
			OptionalFile: &Link{
				File: coal.P(file2),
			},
		}
		err := tester.RunCallback(&fire.Context{
			Operation: fire.Find,
			Model:     model,
			Controller: &fire.Controller{
				Model: coal.Init(&testModel{}),
			},
		}, decorator)
		assert.NoError(t, err)
		assert.NotEmpty(t, model.RequiredFile.ViewKey)
		assert.NotEmpty(t, model.OptionalFile.ViewKey)

		var viewKey1 ViewKey
		err = storage.notary.Verify(&viewKey1, model.RequiredFile.ViewKey)
		assert.NoError(t, err)
		assert.Equal(t, file1, viewKey1.File)

		var viewKey2 ViewKey
		err = storage.notary.Verify(&viewKey2, model.OptionalFile.ViewKey)
		assert.NoError(t, err)
		assert.Equal(t, file2, viewKey2.File)
	})
}

func TestStorageDownload(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		file, err := storage.upload(nil, "foo/bar", 12, strings.NewReader("Hello World!"))
		assert.NoError(t, err)
		assert.NotNil(t, file)

		file.State = Claimed
		tester.Update(file)

		key, err := storage.notary.Issue(&ViewKey{
			Base: heat.Base{},
			File: file.ID(),
		})
		assert.NoError(t, err)
		assert.NotEmpty(t, key)

		action := storage.DownloadAction()

		req := httptest.NewRequest("GET", "/foo?key="+key, nil)
		rec, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, action)
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, rec.Code)
		assert.Equal(t, "foo/bar", rec.Header().Get("Content-Type"))
		assert.Equal(t, "Hello World!", rec.Body.String())
	})
}

func TestStorageCleanup(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		file, err := storage.upload(nil, "foo/bar", 12, strings.NewReader("Hello World!"))
		assert.NoError(t, err)
		assert.NotNil(t, file)

		file.State = Released
		tester.Update(file)

		time.Sleep(10 * time.Millisecond)

		err = storage.Cleanup(nil, 1)
		assert.NoError(t, err)
		assert.Equal(t, 0, tester.Count(&File{}))
	})
}
<commit_msg>removed todo<commit_after>package blaze

import (
	"bytes"
	"mime/multipart"
	"net/http"
	"net/http/httptest"
	"net/textproto"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	"github.com/256dpi/fire"
	"github.com/256dpi/fire/coal"
	"github.com/256dpi/fire/heat"
)

func TestStorageUploadInvalidContentType(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		body := strings.NewReader("Hello World!")
		req := httptest.NewRequest("POST", "/foo", body)

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(0))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusBadRequest, res.Code)
		assert.Equal(t, "", res.Body.String())

		assert.Equal(t, 0, tester.Count(&File{}))
	})
}

func TestStorageUploadRaw(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		service := NewMemory()
		storage := NewStorage(tester.Store, testNotary, service)

		body := strings.NewReader("Hello World!")

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", "application/octet-stream")

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(0))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, res.Code)
		assert.NotEmpty(t, res.Body.String())
		assert.Equal(t, map[string]Blob{
			"1": {
				Type:  "application/octet-stream",
				Bytes: []byte("Hello World!"),
			},
		}, service.Blobs)

		files := *tester.FindAll(&File{}).(*[]*File)
		assert.Len(t, files, 1)
		assert.Equal(t, Uploaded, files[0].State)
		assert.Equal(t, "application/octet-stream", files[0].Type)
		assert.Equal(t, int64(12), files[0].Length)
		assert.Equal(t, Handle{"id": "1"}, files[0].Handle)
	})
}

func TestStorageUploadRawLimit(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		body := strings.NewReader("Hello World!")

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", "application/octet-stream")

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(1))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusRequestEntityTooLarge, res.Code)
		assert.Equal(t, "", res.Body.String())
	})
}

func TestStorageUploadFormFiles(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		service := NewMemory()
		storage := NewStorage(tester.Store, testNotary, service)

		body := &bytes.Buffer{}
		writer := multipart.NewWriter(body)

		part, err := writer.CreateFormFile("file1", "data1.bin")
		assert.NoError(t, err)

		_, err = part.Write([]byte("Hello World 1!"))
		assert.NoError(t, err)

		part, err = writer.CreateFormFile("file2", "data2.bin")
		assert.NoError(t, err)

		_, err = part.Write([]byte("Hello World 2!"))
		assert.NoError(t, err)

		err = writer.Close()
		assert.NoError(t, err)

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", writer.FormDataContentType())

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(0))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, res.Code)
		assert.NotEmpty(t, res.Body.String())

		assert.Equal(t, map[string]Blob{
			"1": {
				Type:  "application/octet-stream",
				Bytes: []byte("Hello World 1!"),
			},
			"2": {
				Type:  "application/octet-stream",
				Bytes: []byte("Hello World 2!"),
			},
		}, service.Blobs)

		files := *tester.FindAll(&File{}).(*[]*File)
		assert.Len(t, files, 2)
		assert.Equal(t, Uploaded, files[0].State)
		assert.Equal(t, "application/octet-stream", files[0].Type)
		assert.Equal(t, int64(14), files[0].Length)
		assert.Equal(t, Handle{"id": "1"}, files[0].Handle)
		assert.Equal(t, Uploaded, files[1].State)
		assert.Equal(t, "application/octet-stream", files[1].Type)
		assert.Equal(t, int64(14), files[1].Length)
		assert.Equal(t, Handle{"id": "2"}, files[1].Handle)
	})
}

func TestStorageUploadFormFilesLimit(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		body := &bytes.Buffer{}
		writer := multipart.NewWriter(body)

		part, err := writer.CreateFormFile("file1", "data1.bin")
		assert.NoError(t, err)

		_, err = part.Write([]byte("Hello World 1!"))
		assert.NoError(t, err)

		err = writer.Close()
		assert.NoError(t, err)

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", writer.FormDataContentType())

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(1))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusRequestEntityTooLarge, res.Code)
		assert.Equal(t, "", res.Body.String())
	})
}

func TestStorageUploadMultipart(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		service := NewMemory()
		storage := NewStorage(tester.Store, testNotary, service)

		body := &bytes.Buffer{}
		writer := multipart.NewWriter(body)

		part, err := writer.CreatePart(textproto.MIMEHeader{
			"Content-Disposition": []string{`form-data; name="file1"; filename="style.css"`},
			"Content-Type":        []string{"text/css"},
		})

		_, err = part.Write([]byte("h1 { color: red; }"))
		assert.NoError(t, err)

		part, err = writer.CreatePart(textproto.MIMEHeader{
			"Content-Disposition": []string{`form-data; name="file2"; filename="script.js"`},
			"Content-Type":        []string{"text/javascript"},
		})

		_, err = part.Write([]byte("console.log('Hello World!);"))
		assert.NoError(t, err)

		err = writer.Close()
		assert.NoError(t, err)

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", writer.FormDataContentType())

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(0))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, res.Code)
		assert.NotEmpty(t, res.Body.String())

		assert.Equal(t, map[string]Blob{
			"1": {
				Type:  "text/css",
				Bytes: []byte("h1 { color: red; }"),
			},
			"2": {
				Type:  "text/javascript",
				Bytes: []byte("console.log('Hello World!);"),
			},
		}, service.Blobs)
	})
}

func TestStorageUploadMultipartLimit(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		body := &bytes.Buffer{}
		writer := multipart.NewWriter(body)

		part, err := writer.CreatePart(textproto.MIMEHeader{
			"Content-Disposition": []string{`form-data; name="file1"; filename="style.css"`},
			"Content-Type":        []string{"text/css"},
		})

		_, err = part.Write([]byte("console.log('Hello World!);"))
		assert.NoError(t, err)

		err = writer.Close()
		assert.NoError(t, err)

		req := httptest.NewRequest("POST", "/foo", body)
		req.Header.Set("Content-Type", writer.FormDataContentType())

		res, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, storage.UploadAction(1))
		assert.NoError(t, err)
		assert.Equal(t, http.StatusRequestEntityTooLarge, res.Code)
		assert.Equal(t, "", res.Body.String())
	})
}

func TestStorageValidator(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		validator := storage.Validator()

		/* missing */

		model := &testModel{}
		err := tester.RunCallback(&fire.Context{
			Operation: fire.Create,
			Model:     model,
			Controller: &fire.Controller{
				Model: coal.Init(&testModel{}),
			},
		}, validator)
		assert.Error(t, err)
		assert.Equal(t, "required-file: missing claim key", err.Error())

		/* required */

		file1 := tester.Save(&File{
			State: Uploaded,
		}).(*File)

		claimKey1, err := storage.notary.Issue(&ClaimKey{
			File: file1.ID(),
		})
		assert.NoError(t, err)
		assert.NotEmpty(t, claimKey1)

		model = &testModel{
			RequiredFile: Link{
				ClaimKey: claimKey1,
			},
		}
		err = tester.RunCallback(&fire.Context{
			Operation: fire.Create,
			Model:     model,
			Controller: &fire.Controller{
				Model: coal.Init(&testModel{}),
			},
		}, validator)
		assert.NoError(t, err)
		assert.Equal(t, file1.ID(), *model.RequiredFile.File)

		file1 = tester.Fetch(&File{}, file1.ID()).(*File)
		assert.Equal(t, Claimed, file1.State)
	})
}

func TestStorageDecorator(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		decorator := storage.Decorator()

		file1 := coal.New()
		file2 := coal.New()

		model := &testModel{
			RequiredFile: Link{
				File: coal.P(file1),
			},
			OptionalFile: &Link{
				File: coal.P(file2),
			},
		}
		err := tester.RunCallback(&fire.Context{
			Operation: fire.Find,
			Model:     model,
			Controller: &fire.Controller{
				Model: coal.Init(&testModel{}),
			},
		}, decorator)
		assert.NoError(t, err)
		assert.NotEmpty(t, model.RequiredFile.ViewKey)
		assert.NotEmpty(t, model.OptionalFile.ViewKey)

		var viewKey1 ViewKey
		err = storage.notary.Verify(&viewKey1, model.RequiredFile.ViewKey)
		assert.NoError(t, err)
		assert.Equal(t, file1, viewKey1.File)

		var viewKey2 ViewKey
		err = storage.notary.Verify(&viewKey2, model.OptionalFile.ViewKey)
		assert.NoError(t, err)
		assert.Equal(t, file2, viewKey2.File)
	})
}

func TestStorageDownload(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		file, err := storage.upload(nil, "foo/bar", 12, strings.NewReader("Hello World!"))
		assert.NoError(t, err)
		assert.NotNil(t, file)

		file.State = Claimed
		tester.Update(file)

		key, err := storage.notary.Issue(&ViewKey{
			Base: heat.Base{},
			File: file.ID(),
		})
		assert.NoError(t, err)
		assert.NotEmpty(t, key)

		action := storage.DownloadAction()

		req := httptest.NewRequest("GET", "/foo?key="+key, nil)
		rec, err := tester.RunAction(&fire.Context{
			HTTPRequest: req,
		}, action)
		assert.NoError(t, err)
		assert.Equal(t, http.StatusOK, rec.Code)
		assert.Equal(t, "foo/bar", rec.Header().Get("Content-Type"))
		assert.Equal(t, "Hello World!", rec.Body.String())
	})
}

func TestStorageCleanup(t *testing.T) {
	withTester(t, func(t *testing.T, tester *fire.Tester) {
		storage := NewStorage(tester.Store, testNotary, NewMemory())

		file, err := storage.upload(nil, "foo/bar", 12, strings.NewReader("Hello World!"))
		assert.NoError(t, err)
		assert.NotNil(t, file)

		file.State = Released
		tester.Update(file)

		time.Sleep(10 * time.Millisecond)

		err = storage.Cleanup(nil, 1)
		assert.NoError(t, err)
		assert.Equal(t, 0, tester.Count(&File{}))
	})
}
<|endoftext|>


<commit_before>package java

import (
	"fmt"
	"io"
	"strings"

	"github.com/anchore/syft/internal/log"

	"github.com/anchore/syft/internal"
	"github.com/anchore/syft/internal/file"
	"github.com/anchore/syft/syft/pkg"
	"github.com/anchore/syft/syft/pkg/cataloger/common"
)

// integrity check
var _ common.ParserFn = parseJavaArchive

var archiveFormatGlobs = []string{
	"**/*.jar",
	"**/*.war",
	"**/*.ear",
	"**/*.jpi",
	"**/*.hpi",
}

type archiveParser struct {
	discoveredPkgs internal.StringSet
	fileManifest   file.ZipFileManifest
	virtualPath    string
	archivePath    string
	contentPath    string
	fileInfo       archiveFilename
	detectNested   bool
}

// parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
	parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
	// note: even on error, we should always run cleanup functions
	defer cleanupFn()
	if err != nil {
		return nil, err
	}
	return parser.parse()
}

// uniquePkgKey creates a unique string to identify the given package.
func uniquePkgKey(p *pkg.Package) string {
	if p == nil {
		return ""
	}
	return fmt.Sprintf("%s|%s", p.Name, p.Version)
}

// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover
// and parse nested archives or ignore them.
func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested bool) (*archiveParser, func(), error) {
	contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(reader)
	if err != nil {
		return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
	}

	fileManifest, err := file.NewZipFileManifest(archivePath)
	if err != nil {
		return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
	}

	// fetch the last element of the virtual path
	virtualElements := strings.Split(virtualPath, ":")
	currentFilepath := virtualElements[len(virtualElements)-1]

	return &archiveParser{
		discoveredPkgs: internal.NewStringSet(),
		fileManifest:   fileManifest,
		virtualPath:    virtualPath,
		archivePath:    archivePath,
		contentPath:    contentPath,
		fileInfo:       newJavaArchiveFilename(currentFilepath),
		detectNested:   detectNested,
	}, cleanupFn, nil
}

// parse the loaded archive and return all packages found.
func (j *archiveParser) parse() ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	// find the parent package from the java manifest
	parentPkg, err := j.discoverMainPackage()
	if err != nil {
		return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
	}

	// don't add the parent package yet, we still may discover aux info to add to the metadata (but still track it as added to prevent duplicates)
	parentKey := uniquePkgKey(parentPkg)
	if parentKey != "" {
		j.discoveredPkgs.Add(parentKey)
	}

	// find aux packages from pom.properties
	auxPkgs, err := j.discoverPkgsFromAllPomProperties(parentPkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, auxPkgs...)

	// find nested java archive packages
	nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, nestedPkgs...)

	// lastly, add the parent package to the list (assuming the parent exists)
	if parentPkg != nil {
		// only the parent package gets the type, nested packages may be of a different package type (or not of a package type at all, since they may not be bundled)
		parentPkg.Type = j.fileInfo.pkgType()
		pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
	}

	return pkgs, nil
}

// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
	// search and parse java manifest files
	manifestMatches := j.fileManifest.GlobMatch(manifestGlob)
	if len(manifestMatches) > 1 {
		return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
	} else if len(manifestMatches) == 0 {
		// we did not find any manifests, but that may not be a problem (there may be other information to generate packages for)
		return nil, nil
	}

	// fetch the manifest file
	contents, err := file.ContentsFromZip(j.archivePath, manifestMatches...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.virtualPath, err)
	}

	// parse the manifest file into a rich object
	manifestContents := contents[manifestMatches[0]]
	manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
	if err != nil {
		log.Warnf("failed to parse java manifest (%s): %+v", j.virtualPath, err)
		return nil, nil
	}

	return &pkg.Package{
		Name:         selectName(manifest, j.fileInfo),
		Version:      selectVersion(manifest, j.fileInfo),
		Language:     pkg.Java,
		Type:         pkg.JavaPkg,
		MetadataType: pkg.JavaMetadataType,
		Metadata: pkg.JavaMetadata{
			VirtualPath: j.virtualPath,
			Manifest:    manifest,
		},
	}, nil
}

// discoverPkgsFromAllPomProperties parses Maven POM properties for a given parent package, returning all listed Java packages found for each pom properties discovered.
func (j *archiveParser) discoverPkgsFromAllPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	// search and parse pom.properties files & fetch the contents
	contents, err := file.ContentsFromZip(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob)...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract pom.properties: %w", err)
	}

	// parse the manifest file into a rich object
	for propsPath, propsContents := range contents {
		propsObj, err := parsePomProperties(propsPath, strings.NewReader(propsContents))
		if err != nil {
			log.Warnf("failed to parse pom.properties (%s): %+v", j.virtualPath, err)
			continue
		}

		if propsObj == nil {
			continue
		}

		if propsObj.Version == "" || propsObj.ArtifactID == "" {
			// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
			continue
		}

		if parentPkg == nil {
			continue
		}

		pkgs = append(pkgs, j.packagesFromPomProperties(propsObj, parentPkg)...)
	}
	return pkgs, nil
}

// packagesFromPomProperties processes a single Maven POM properties for a given parent package, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) packagesFromPomProperties(propsObj *pkg.PomProperties, parentPkg *pkg.Package) (pkgs []pkg.Package) {
	parentKey := uniquePkgKey(parentPkg)

	// keep the artifact name within the virtual path if this package does not match the parent package
	vPathSuffix := ""
	if !strings.HasPrefix(propsObj.ArtifactID, parentPkg.Name) {
		vPathSuffix += ":" + propsObj.ArtifactID
	}
	virtualPath := j.virtualPath + vPathSuffix

	// discovered props = new package
	p := pkg.Package{
		Name:         propsObj.ArtifactID,
		Version:      propsObj.Version,
		Language:     pkg.Java,
		Type:         pkg.JavaPkg,
		MetadataType: pkg.JavaMetadataType,
		Metadata: pkg.JavaMetadata{
			VirtualPath:   virtualPath,
			PomProperties: propsObj,
			Parent:        parentPkg,
		},
	}

	pkgKey := uniquePkgKey(&p)

	// the name/version pair matches...
	matchesParentPkg := pkgKey == parentKey

	// the virtual path matches...
	matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath

	// the pom artifactId has the parent name or vice versa
	if propsObj.ArtifactID != "" {
		matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, propsObj.ArtifactID) || strings.Contains(propsObj.ArtifactID, parentPkg.Name)
	}

	if matchesParentPkg {
		// we've run across more information about our parent package, add this info to the parent package metadata
		// the pom properties is typically a better source of information for name and version than the manifest
		if parentPkg.Name == "" {
			parentPkg.Name = p.Name
		}
		if parentPkg.Version == "" {
			parentPkg.Version = p.Version
		}

		// keep the pom properties, but don't overwrite existing pom properties
		parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
		if ok && parentMetadata.PomProperties == nil {
			parentMetadata.PomProperties = propsObj
			parentPkg.Metadata = parentMetadata
		}
	}

	if !matchesParentPkg && !j.discoveredPkgs.Contains(pkgKey) {
		// only keep packages we haven't seen yet (and are not related to the parent package)
		pkgs = append(pkgs, p)
	}
	return pkgs
}

// discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	if !j.detectNested {
		return pkgs, nil
	}

	// search and parse pom.properties files & fetch the contents
	openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract files from zip: %w", err)
	}

	// discover nested artifacts
	for archivePath, archiveOpener := range openers {
		archiveReadCloser, err := archiveOpener.Open()
		if err != nil {
			return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
		}
		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
		if err != nil {
			if closeErr := archiveReadCloser.Close(); closeErr != nil {
				log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
			}
			return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
		}
		if err = archiveReadCloser.Close(); err != nil {
			return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
		}

		// attach the parent package to all discovered packages that are not already associated with a java archive
		for _, p := range nestedPkgs {
			if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok {
				if metadata.Parent == nil {
					metadata.Parent = parentPkg
				}
				p.Metadata = metadata
			}
			pkgs = append(pkgs, p)
		}
	}

	return pkgs, nil
}
<commit_msg>Refactor Java archive parsing logic<commit_after>package java

import (
	"fmt"
	"io"
	"strings"

	"github.com/anchore/syft/internal/log"

	"github.com/anchore/syft/internal/file"
	"github.com/anchore/syft/syft/pkg"
	"github.com/anchore/syft/syft/pkg/cataloger/common"
)

// integrity check
var _ common.ParserFn = parseJavaArchive

var archiveFormatGlobs = []string{
	"**/*.jar",
	"**/*.war",
	"**/*.ear",
	"**/*.jpi",
	"**/*.hpi",
}

type archiveParser struct {
	fileManifest file.ZipFileManifest
	virtualPath  string
	archivePath  string
	contentPath  string
	fileInfo     archiveFilename
	detectNested bool
}

// parseJavaArchive is a parser function for java archive contents, returning all Java libraries and nested archives.
func parseJavaArchive(virtualPath string, reader io.Reader) ([]pkg.Package, error) {
	parser, cleanupFn, err := newJavaArchiveParser(virtualPath, reader, true)
	// note: even on error, we should always run cleanup functions
	defer cleanupFn()
	if err != nil {
		return nil, err
	}
	return parser.parse()
}

// uniquePkgKey creates a unique string to identify the given package.
func uniquePkgKey(p *pkg.Package) string {
	if p == nil {
		return ""
	}
	return fmt.Sprintf("%s|%s", p.Name, p.Version)
}

// newJavaArchiveParser returns a new java archive parser object for the given archive. Can be configured to discover
// and parse nested archives or ignore them.
func newJavaArchiveParser(virtualPath string, reader io.Reader, detectNested bool) (*archiveParser, func(), error) {
	contentPath, archivePath, cleanupFn, err := saveArchiveToTmp(reader)
	if err != nil {
		return nil, cleanupFn, fmt.Errorf("unable to process java archive: %w", err)
	}

	fileManifest, err := file.NewZipFileManifest(archivePath)
	if err != nil {
		return nil, cleanupFn, fmt.Errorf("unable to read files from java archive: %w", err)
	}

	// fetch the last element of the virtual path
	virtualElements := strings.Split(virtualPath, ":")
	currentFilepath := virtualElements[len(virtualElements)-1]

	return &archiveParser{
		fileManifest: fileManifest,
		virtualPath:  virtualPath,
		archivePath:  archivePath,
		contentPath:  contentPath,
		fileInfo:     newJavaArchiveFilename(currentFilepath),
		detectNested: detectNested,
	}, cleanupFn, nil
}

// parse the loaded archive and return all packages found.
func (j *archiveParser) parse() ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	// find the parent package from the java manifest
	parentPkg, err := j.discoverMainPackage()
	if err != nil {
		return nil, fmt.Errorf("could not generate package from %s: %w", j.virtualPath, err)
	}

	// find aux packages from pom.properties and potentially modify the existing parentPkg
	auxPkgs, err := j.discoverPkgsFromAllPomProperties(parentPkg)
	if err != nil {
		return nil, err
	}
	pkgs = append(pkgs, auxPkgs...)

	if j.detectNested {
		// find nested java archive packages
		nestedPkgs, err := j.discoverPkgsFromNestedArchives(parentPkg)
		if err != nil {
			return nil, err
		}
		pkgs = append(pkgs, nestedPkgs...)
	}

	// lastly, add the parent package to the list (assuming the parent exists)
	if parentPkg != nil {
		// only the parent package gets the type, nested packages may be of a different package type (or not of a package type at all, since they may not be bundled)
		parentPkg.Type = j.fileInfo.pkgType()
		pkgs = append([]pkg.Package{*parentPkg}, pkgs...)
	}

	return pkgs, nil
}

// discoverMainPackage parses the root Java manifest used as the parent package to all discovered nested packages.
func (j *archiveParser) discoverMainPackage() (*pkg.Package, error) {
	// search and parse java manifest files
	manifestMatches := j.fileManifest.GlobMatch(manifestGlob)
	if len(manifestMatches) > 1 {
		return nil, fmt.Errorf("found multiple manifests in the jar: %+v", manifestMatches)
	} else if len(manifestMatches) == 0 {
		// we did not find any manifests, but that may not be a problem (there may be other information to generate packages for)
		return nil, nil
	}

	// fetch the manifest file
	contents, err := file.ContentsFromZip(j.archivePath, manifestMatches...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract java manifests (%s): %w", j.virtualPath, err)
	}

	// parse the manifest file into a rich object
	manifestContents := contents[manifestMatches[0]]
	manifest, err := parseJavaManifest(j.archivePath, strings.NewReader(manifestContents))
	if err != nil {
		log.Warnf("failed to parse java manifest (%s): %+v", j.virtualPath, err)
		return nil, nil
	}

	return &pkg.Package{
		Name:         selectName(manifest, j.fileInfo),
		Version:      selectVersion(manifest, j.fileInfo),
		Language:     pkg.Java,
		Type:         pkg.JavaPkg,
		MetadataType: pkg.JavaMetadataType,
		Metadata: pkg.JavaMetadata{
			VirtualPath: j.virtualPath,
			Manifest:    manifest,
		},
	}, nil
}

// discoverPkgsFromAllPomProperties parses Maven POM properties for a given
// parent package, returning all listed Java packages found for each pom
// properties discovered and potentially updating the given parentPkg with new
// data.
func (j *archiveParser) discoverPkgsFromAllPomProperties(parentPkg *pkg.Package) ([]pkg.Package, error) {
	if parentPkg == nil {
		return nil, nil
	}

	var pkgs []pkg.Package

	// search and parse pom.properties files & fetch the contents
	contentsOfPomPropertiesFiles, err := file.ContentsFromZip(j.archivePath, j.fileManifest.GlobMatch(pomPropertiesGlob)...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract pom.properties: %w", err)
	}

	for filePath, fileContents := range contentsOfPomPropertiesFiles {
		// parse the pom properties file into a rich object
		pomProperties, err := parsePomProperties(filePath, strings.NewReader(fileContents))
		if err != nil {
			log.Warnf("failed to parse pom.properties (%s): %+v", j.virtualPath, err)
			continue
		}

		if pomProperties == nil {
			continue
		}

		if pomProperties.Version == "" || pomProperties.ArtifactID == "" {
			// TODO: if there is no parentPkg (no java manifest) one of these poms could be the parent. We should discover the right parent and attach the correct info accordingly to each discovered package
			continue
		}

		pkgs = append(pkgs, j.packagesFromPomProperties(pomProperties, parentPkg)...)
	}
	return pkgs, nil
}

// packagesFromPomProperties processes a single Maven POM properties for a given parent package, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) packagesFromPomProperties(pomProperties *pkg.PomProperties, parentPkg *pkg.Package) []pkg.Package {
	// keep the artifact name within the virtual path if this package does not match the parent package
	vPathSuffix := ""
	if !strings.HasPrefix(pomProperties.ArtifactID, parentPkg.Name) {
		vPathSuffix += ":" + pomProperties.ArtifactID
	}
	virtualPath := j.virtualPath + vPathSuffix

	// discovered props = new package
	p := pkg.Package{
		Name:         pomProperties.ArtifactID,
		Version:      pomProperties.Version,
		Language:     pkg.Java,
		Type:         pkg.JavaPkg,
		MetadataType: pkg.JavaMetadataType,
		Metadata: pkg.JavaMetadata{
			VirtualPath:   virtualPath,
			PomProperties: pomProperties,
			Parent:        parentPkg,
		},
	}

	pkgKey := uniquePkgKey(&p)
	parentKey := uniquePkgKey(parentPkg)

	// the name/version pair matches...
	matchesParentPkg := pkgKey == parentKey

	// the virtual path matches...
	matchesParentPkg = matchesParentPkg || parentPkg.Metadata.(pkg.JavaMetadata).VirtualPath == virtualPath

	// the pom artifactId has the parent name or vice versa
	if pomProperties.ArtifactID != "" {
		matchesParentPkg = matchesParentPkg || strings.Contains(parentPkg.Name, pomProperties.ArtifactID) || strings.Contains(pomProperties.ArtifactID, parentPkg.Name)
	}

	if !matchesParentPkg {
		// only keep packages we haven't seen yet (and are not related to the parent package)
		return []pkg.Package{p}
	}

	// we've run across more information about our parent package, add this info to the parent package metadata
	// the pom properties is typically a better source of information for name and version than the manifest
	if parentPkg.Name == "" {
		parentPkg.Name = p.Name
	}
	if parentPkg.Version == "" {
		parentPkg.Version = p.Version
	}

	// keep the pom properties, but don't overwrite existing pom properties
	parentMetadata, ok := parentPkg.Metadata.(pkg.JavaMetadata)
	if ok && parentMetadata.PomProperties == nil {
		parentMetadata.PomProperties = pomProperties
		parentPkg.Metadata = parentMetadata
	}

	return nil
}

// discoverPkgsFromNestedArchives finds Java archives within Java archives, returning all listed Java packages found and
// associating each discovered package to the given parent package.
func (j *archiveParser) discoverPkgsFromNestedArchives(parentPkg *pkg.Package) ([]pkg.Package, error) {
	var pkgs = make([]pkg.Package, 0)

	// search and parse pom.properties files & fetch the contents
	openers, err := file.ExtractFromZipToUniqueTempFile(j.archivePath, j.contentPath, j.fileManifest.GlobMatch(archiveFormatGlobs...)...)
	if err != nil {
		return nil, fmt.Errorf("unable to extract files from zip: %w", err)
	}

	// discover nested artifacts
	for archivePath, archiveOpener := range openers {
		archiveReadCloser, err := archiveOpener.Open()
		if err != nil {
			return nil, fmt.Errorf("unable to open archived file from tempdir: %w", err)
		}
		nestedPath := fmt.Sprintf("%s:%s", j.virtualPath, archivePath)
		nestedPkgs, err := parseJavaArchive(nestedPath, archiveReadCloser)
		if err != nil {
			if closeErr := archiveReadCloser.Close(); closeErr != nil {
				log.Warnf("unable to close archived file from tempdir: %+v", closeErr)
			}
			return nil, fmt.Errorf("unable to process nested java archive (%s): %w", archivePath, err)
		}
		if err = archiveReadCloser.Close(); err != nil {
			return nil, fmt.Errorf("unable to close archived file from tempdir: %w", err)
		}

		// attach the parent package to all discovered packages that are not already associated with a java archive
		for _, p := range nestedPkgs {
			if metadata, ok := p.Metadata.(pkg.JavaMetadata); ok {
				if metadata.Parent == nil {
					metadata.Parent = parentPkg
				}
				p.Metadata = metadata
			}
			pkgs = append(pkgs, p)
		}
	}

	return pkgs, nil
}
<|endoftext|>


<commit_before>package lib

import (
	"bufio"
	"fmt"
	"io/ioutil"
	n "net"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"subutai/config"
	"subutai/lib/net"
	"subutai/log"
	"syscall"
)

func LxcManagementNetwork(args []string) {
	if len(args) < 3 {
		log.Error("Not enough arguments")
	}
	switch args[2] {
	case "-s", "--showflow":
		showFlow(args[3])
	case "-p", "--showport":
		showPort(args[3])
	case "-L", "--listn2n":
		listOpenedN2NTunnel()
	case "-D", "--deletegateway":
		deleteGateway(args[3])
	case "-S", "--listopenedtab":
		listOpenedTapDevice()
	case "-V", "--removetab":
		removeTapDevice(args[3])
	case "-v", "--listvnimap":
		listVNIMap()
	case "-r", "--removetunnel":
		removeTunnel(args[3])
	case "-e", "--reloadn2n":
		reloadN2N(args[3], args[4])
	case "-T", "--creategateway":
		createGateway(args[3], args[4])
	case "-M", "--removevni":
		delVNI(args[3], args[4], args[5])
	case "-R", "--removen2n":
		removeN2NTunnel(args[3], args[4])
	case "-E", "--reservvni":
		reservVNI(args[3], args[4], args[5])
	case "-m", "--createvnimap":
		createVNIMap(args[3], args[4], args[5], args[6])
	case "-c", "--createtunnel":
		log.Check(log.FatalLevel, "create tunnel", createTunnel(args[3], args[4], args[5]))
	case "-f", "--addflow":
		addFlow(args[3], args[4])
		log.Info("Flow configuration added")
	case "-l", "--listtunnel":
		liste := listTunnel()
		fmt.Println("List of Tunnels\n--------")
		for _, v := range liste {
			fmt.Println(string(v))
		}
	case "-d", "--deleteflow":
		if len(args)-3 < 2 {
			deleteFlow(args[3], "")
		} else {
			deleteFlow(args[3], args[4])
		}
	case "-N", "--addn2n":
		p2pTunnel(args[5], args[6], args[7])
		// if len(args)-3 == 8 {
		// func createN2NTunnel(interfaceName, communityName, localPeepIPAddr) {
		// func createN2NTunnel(superNodeIPaddr, superNodePort, interfaceName, communityName,localPeepIPAddr, keyType, keyFile, managementPort string) {
		// createN2NTunnel(args[3], args[4], args[5], args[6], args[7], args[8], args[9], args[10])
		// } else if len(args)-3 == 7 { // management port can be empty
		// createN2NTunnel(args[3], args[4], args[5], args[6], args[7], args[8], args[9], "")
		// } else {
		// log.Warn("please check you have given all arguments correctly")
		// }
	case "-Z", "--vniop":
		switch args[3] {
		case "deleteall":
			net.DeleteAllVNI(args[4])
			deleteGateway(args[4])
		case "delete":
			net.DeleteVNI(args[4], args[5], args[6])
		case "list":
			net.ListVNI()
		}
	}
}

func p2pFile(line string) {
	path := config.Agent.DataPrefix + "/var/subutai-network/"
	file := path + "p2p.txt"
	if _, err := os.Stat(path); os.IsNotExist(err) {
		log.Check(log.FatalLevel, "create "+path+" folder", os.MkdirAll(path, 0755))
	}
	if _, err := os.Stat(file); os.IsNotExist(err) {
		_, err = os.Create(file)
		log.Check(log.FatalLevel, "Creating "+file, err)
	}

	f, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
	log.Check(log.FatalLevel, "Opening file for append "+file, err)
	defer f.Close()
	_, err = f.WriteString(line + "\n")
	log.Check(log.FatalLevel, "Opening file for append "+file, err)
}

func p2pTunnel(interfaceName, communityName, localPeepIPAddr string) {
	p2pFile(interfaceName + " " + localPeepIPAddr + " " + communityName)
	log.Check(log.FatalLevel, "p2p command: ", exec.Command("p2p", "-dev", interfaceName, "-ip", localPeepIPAddr, "-hash", communityName).Start())
}

func createTunnel(tunnelPortName, tunnelIPAddress, tunnelType string) error {
	log.Info("tunnel port name: " + tunnelPortName)
	log.Info("tunnel IP address: " + tunnelIPAddress)
	log.Info("tunnel type: " + tunnelType)
	log.Check(log.FatalLevel, "check tunnel validity ", net.CheckTunnelPortNameValidity(tunnelPortName))
	log.Check(log.FatalLevel, "check ip validity "+tunnelIPAddress, net.CheckIPValidity(listTunnel(), tunnelIPAddress))
	if tunnelType == "vxlan" || tunnelType == "gre" {
		log.Check(log.FatalLevel, "create tunnel ", net.CreateTunnel(listTunnel(), tunnelPortName, tunnelIPAddress, tunnelType))
	} else {
		log.Error("Tunnel type must be vxlan or gre")
	}
	return nil
}
func listTunnel() []string { // we need a list when ip address checking.
	var returnArr []string
	list := net.ListTunnels()
	listA := strings.Split(string(list), "\n")

	for k, v := range listA {
		if strings.Contains(string(v), "remote_ip") {
			devInt := strings.Fields(listA[k-2])
			strLine := strings.Trim(devInt[1], "\"")
			devIP := strings.Fields(v)
			strLine = strLine + "-" + strings.Trim(strings.Trim(devIP[2], "remote_ip="), "\"}")
			returnArr = append(returnArr, strLine)
		}
	}
	return returnArr
}
func removeTunnel(tunnelPortName string) {
	retVal := net.CheckTunnelPortNameValidity(tunnelPortName)
	log.Check(log.WarnLevel, " remove "+tunnelPortName+"_vni_vlan",
		os.Remove(config.Agent.DataPrefix+"var/subutai-network/"+tunnelPortName+"_vni_vlan"))

	// basically it return err if given tunnelPortName exits.
	if retVal != nil {
		log.Info(tunnelPortName + " found in system.")
		log.Check(log.FatalLevel, "remove tunnel", net.RemovePort(tunnelPortName))
		log.Info(tunnelPortName + " removed")
	} else {
		log.Info(tunnelPortName + " not exists in system so NOT to remove")
	}

}

func addFlow(bridgeName, flowConfiguration string) {
	log.Info("flow configuration adding with: " + bridgeName + " " + flowConfiguration)
	log.Check(log.FatalLevel, "addFlow: ", net.AddFlowConfig(bridgeName, flowConfiguration))
}

func showFlow(bridgeName string) {
	s, err := net.DumpBridge(bridgeName)
	if err != nil {
		log.Error("showFlow " + string(s))
	}
	log.Info("Flow Table informations of " + bridgeName)
	fmt.Println(s) // uufff...
}

func showPort(bridgeName string) {
	s, err := net.DumpPort(bridgeName)
	if err != nil {
		log.Error("showPort ", string(s))
	}
	log.Info("Port informations of " + bridgeName)
	fmt.Println(s)
}
func deleteFlow(bridgeName, matchCase string) {
	log.Check(log.FatalLevel, "deleteflow ", net.DeleteFlow(bridgeName, matchCase))
	log.Info(bridgeName + " flows are deleted according to " + matchCase)
}

// refer to n2n.go
func createN2NTunnel(superNodeIPaddr, superNodePort, interfaceName, communityName,
	localPeepIPAddr, keyType, keyFile, managementPort string) {
	// check: if there is /var/subutai-network/edgePorts.txt
	log.Info("n2n create tunnel started")
	log.Info("superNode IP: " + superNodeIPaddr)
	log.Info("superNode port: " + superNodePort)
	log.Info("interface name: " + interfaceName)
	log.Info("community name: " + communityName)
	log.Info("local peer IP: " + localPeepIPAddr)
	log.Info("key type: " + keyType)
	log.Info("key file: " + keyFile)
	log.Info("management port: " + managementPort)
	net.CheckOrCreateEdgePortFile()
	// check: if port is given or available.

	if managementPort == "" {
	CutTheLoop:
		for i := 5645; i < 65535; i++ {
			_, err := n.Listen("udp", ":"+string(i))
			if err != nil {
				// then this port is empty.
				managementPort = strconv.Itoa(i)
				break CutTheLoop // cut the loop
			}
		}
	} else if _, err := n.Listen("udp", managementPort); err == nil {
		// ups this is port is being used...
		log.Error("port is used by other process. " + err.Error())
	}
	if managementPort == "" {
		// still??? then there is no port left
		log.Error("no available port in computer")
	}
	net.ProcessEdge(superNodeIPaddr, superNodePort, interfaceName, communityName,
		localPeepIPAddr, keyType, keyFile, managementPort)

}

func listOpenedN2NTunnel() {
	net.PrintN2NTunnels()
}

func removeN2NTunnel(interfaceName, communityName string) {
	pid := net.ReturnPID(interfaceName, communityName)
	i, _ := strconv.Atoi(pid)
	newconf := ""
	log.Check(log.FatalLevel, "remove n2n tunnel: ", syscall.Kill(i, syscall.SIGHUP))

	file, err := os.Open(config.Agent.DataPrefix + "/var/subutai-network/p2p.txt")
	log.Check(log.FatalLevel, "Opening p2p.txt", err)
	scanner := bufio.NewScanner(bufio.NewReader(file))

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, interfaceName) && strings.HasSuffix(line, communityName) {
			newconf = newconf + line
		}
	}
	file.Close()
	log.Check(log.FatalLevel, "Removing p2p tunnel", ioutil.WriteFile(config.Agent.DataPrefix+"/var/subutai-network/p2p.txt", []byte(newconf), 0644))
}

func createGateway(vlanip, vlanid string) {
	net.CreateGateway(vlanip, vlanid)
	log.Info("gateway created")
}

func deleteGateway(vlanid string) {
	net.DeleteGateway(vlanid)
	log.Info("gateway " + vlanid + " removed")
}

func listOpenedTapDevice() {
	net.ListTapDevice()
}

func removeTapDevice(interfaceName string) {
	pid := net.ReturnPID(interfaceName, " ") // i don't know if this will work..
	i, _ := strconv.Atoi(pid)
	log.Check(log.FatalLevel, "remove tap device: ", syscall.Kill(i, syscall.SIGHUP))
	log.Info(interfaceName + " tap device removed")
}

func createVNIMap(tunnelPortName, vni, vlan, envid string) {
	// check: if there is vni file
	if _, err := os.Stat(config.Agent.DataPrefix + "/var/subutai-network/vni_reserve"); os.IsNotExist(err) {
		log.Error("Do Reserve first. No reserved VNIs, not exist file for reserved VNI")
	}
	net.CreateVNIFile(tunnelPortName + "_vni_vlan")
	// check: control if there is such entry in nvi_reserv file.
	ret, _ := net.CheckVNIFile(tunnelPortName+"_vni_vlan", vni, vlan, envid)
	if ret[0] == true {
		log.Info("vni found")
	}
	if ret[1] == true {
		log.Info("vlanid found")
	}
	if ret[2] == true {
		log.Info("envid found")
	}
	if ret[3] == true {
		log.Info("reservation found")
	}

	net.MakeVNIMap(tunnelPortName, vni, vlan, envid)
	log.Info("vni map created: " + vni + " " + vlan + " " + envid)
}

func listVNIMap() {
	if _, err := os.Stat(config.Agent.DataPrefix + "/var/subutai-network/"); os.IsNotExist(err) {
		log.Error("folder not found" + err.Error())
	}
	net.DisplayVNIMap()
}

func delVNI(tunnelPortName, vni, vlan string) {
	if _, err := os.Stat(config.Agent.DataPrefix + "/var/subutai-network/"); os.IsNotExist(err) {
		log.Error("folder not found" + err.Error())
	}
	net.DelVNI(tunnelPortName, vni, vlan)
	log.Info(vni + " " + vlan + " deleted from " + tunnelPortName)
}

func reservVNI(vni, vlan, envid string) {
	// check: create vni file
	net.CreateVNIFile("vni_reserve")
	net.MakeReservation(vni, vlan, envid)
	log.Info(vni + " " + vlan + " " + envid + " is reserved")

}

func reloadN2N(interfaceName, communityName string) {
	net.CheckOrCreateEdgePortFile()
	net.ReloadN2N(interfaceName, communityName)
	log.Info(interfaceName + " " + communityName + " reloaded")
}
<commit_msg>P2P removing fix.. SS-4083<commit_after>package lib

import (
	"bufio"
	"fmt"
	"io/ioutil"
	n "net"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"subutai/config"
	"subutai/lib/net"
	"subutai/log"
	"syscall"
)

func LxcManagementNetwork(args []string) {
	if len(args) < 3 {
		log.Error("Not enough arguments")
	}
	switch args[2] {
	case "-s", "--showflow":
		showFlow(args[3])
	case "-p", "--showport":
		showPort(args[3])
	case "-L", "--listn2n":
		listOpenedN2NTunnel()
	case "-D", "--deletegateway":
		deleteGateway(args[3])
	case "-S", "--listopenedtab":
		listOpenedTapDevice()
	case "-V", "--removetab":
		removeTapDevice(args[3])
	case "-v", "--listvnimap":
		listVNIMap()
	case "-r", "--removetunnel":
		removeTunnel(args[3])
	case "-e", "--reloadn2n":
		reloadN2N(args[3], args[4])
	case "-T", "--creategateway":
		createGateway(args[3], args[4])
	case "-M", "--removevni":
		delVNI(args[3], args[4], args[5])
	case "-R", "--removen2n":
		removeN2NTunnel(args[3], args[4])
	case "-E", "--reservvni":
		reservVNI(args[3], args[4], args[5])
	case "-m", "--createvnimap":
		createVNIMap(args[3], args[4], args[5], args[6])
	case "-c", "--createtunnel":
		log.Check(log.FatalLevel, "create tunnel", createTunnel(args[3], args[4], args[5]))
	case "-f", "--addflow":
		addFlow(args[3], args[4])
		log.Info("Flow configuration added")
	case "-l", "--listtunnel":
		liste := listTunnel()
		fmt.Println("List of Tunnels\n--------")
		for _, v := range liste {
			fmt.Println(string(v))
		}
	case "-d", "--deleteflow":
		if len(args)-3 < 2 {
			deleteFlow(args[3], "")
		} else {
			deleteFlow(args[3], args[4])
		}
	case "-N", "--addn2n":
		p2pTunnel(args[5], args[6], args[7])
		// if len(args)-3 == 8 {
		// func createN2NTunnel(interfaceName, communityName, localPeepIPAddr) {
		// func createN2NTunnel(superNodeIPaddr, superNodePort, interfaceName, communityName,localPeepIPAddr, keyType, keyFile, managementPort string) {
		// createN2NTunnel(args[3], args[4], args[5], args[6], args[7], args[8], args[9], args[10])
		// } else if len(args)-3 == 7 { // management port can be empty
		// createN2NTunnel(args[3], args[4], args[5], args[6], args[7], args[8], args[9], "")
		// } else {
		// log.Warn("please check you have given all arguments correctly")
		// }
	case "-Z", "--vniop":
		switch args[3] {
		case "deleteall":
			net.DeleteAllVNI(args[4])
			deleteGateway(args[4])
		case "delete":
			net.DeleteVNI(args[4], args[5], args[6])
		case "list":
			net.ListVNI()
		}
	}
}

func p2pFile(line string) {
	path := config.Agent.DataPrefix + "/var/subutai-network/"
	file := path + "p2p.txt"
	if _, err := os.Stat(path); os.IsNotExist(err) {
		log.Check(log.FatalLevel, "create "+path+" folder", os.MkdirAll(path, 0755))
	}
	if _, err := os.Stat(file); os.IsNotExist(err) {
		_, err = os.Create(file)
		log.Check(log.FatalLevel, "Creating "+file, err)
	}

	f, err := os.OpenFile(file, os.O_APPEND|os.O_WRONLY, 0600)
	log.Check(log.FatalLevel, "Opening file for append "+file, err)
	defer f.Close()
	_, err = f.WriteString(line + "\n")
	log.Check(log.FatalLevel, "Opening file for append "+file, err)
}

func p2pTunnel(interfaceName, communityName, localPeepIPAddr string) {
	p2pFile(interfaceName + " " + localPeepIPAddr + " " + communityName)
	log.Check(log.FatalLevel, "p2p command: ", exec.Command("p2p", "-dev", interfaceName, "-ip", localPeepIPAddr, "-hash", communityName).Start())
}

func createTunnel(tunnelPortName, tunnelIPAddress, tunnelType string) error {
	log.Info("tunnel port name: " + tunnelPortName)
	log.Info("tunnel IP address: " + tunnelIPAddress)
	log.Info("tunnel type: " + tunnelType)
	log.Check(log.FatalLevel, "check tunnel validity ", net.CheckTunnelPortNameValidity(tunnelPortName))
	log.Check(log.FatalLevel, "check ip validity "+tunnelIPAddress, net.CheckIPValidity(listTunnel(), tunnelIPAddress))
	if tunnelType == "vxlan" || tunnelType == "gre" {
		log.Check(log.FatalLevel, "create tunnel ", net.CreateTunnel(listTunnel(), tunnelPortName, tunnelIPAddress, tunnelType))
	} else {
		log.Error("Tunnel type must be vxlan or gre")
	}
	return nil
}
func listTunnel() []string { // we need a list when ip address checking.
	var returnArr []string
	list := net.ListTunnels()
	listA := strings.Split(string(list), "\n")

	for k, v := range listA {
		if strings.Contains(string(v), "remote_ip") {
			devInt := strings.Fields(listA[k-2])
			strLine := strings.Trim(devInt[1], "\"")
			devIP := strings.Fields(v)
			strLine = strLine + "-" + strings.Trim(strings.Trim(devIP[2], "remote_ip="), "\"}")
			returnArr = append(returnArr, strLine)
		}
	}
	return returnArr
}
func removeTunnel(tunnelPortName string) {
	retVal := net.CheckTunnelPortNameValidity(tunnelPortName)
	log.Check(log.WarnLevel, " remove "+tunnelPortName+"_vni_vlan",
		os.Remove(config.Agent.DataPrefix+"var/subutai-network/"+tunnelPortName+"_vni_vlan"))

	// basically it return err if given tunnelPortName exits.
	if retVal != nil {
		log.Info(tunnelPortName + " found in system.")
		log.Check(log.FatalLevel, "remove tunnel", net.RemovePort(tunnelPortName))
		log.Info(tunnelPortName + " removed")
	} else {
		log.Info(tunnelPortName + " not exists in system so NOT to remove")
	}

}

func addFlow(bridgeName, flowConfiguration string) {
	log.Info("flow configuration adding with: " + bridgeName + " " + flowConfiguration)
	log.Check(log.FatalLevel, "addFlow: ", net.AddFlowConfig(bridgeName, flowConfiguration))
}

func showFlow(bridgeName string) {
	s, err := net.DumpBridge(bridgeName)
	if err != nil {
		log.Error("showFlow " + string(s))
	}
	log.Info("Flow Table informations of " + bridgeName)
	fmt.Println(s) // uufff...
}

func showPort(bridgeName string) {
	s, err := net.DumpPort(bridgeName)
	if err != nil {
		log.Error("showPort ", string(s))
	}
	log.Info("Port informations of " + bridgeName)
	fmt.Println(s)
}
func deleteFlow(bridgeName, matchCase string) {
	log.Check(log.FatalLevel, "deleteflow ", net.DeleteFlow(bridgeName, matchCase))
	log.Info(bridgeName + " flows are deleted according to " + matchCase)
}

// refer to n2n.go
func createN2NTunnel(superNodeIPaddr, superNodePort, interfaceName, communityName,
	localPeepIPAddr, keyType, keyFile, managementPort string) {
	// check: if there is /var/subutai-network/edgePorts.txt
	log.Info("n2n create tunnel started")
	log.Info("superNode IP: " + superNodeIPaddr)
	log.Info("superNode port: " + superNodePort)
	log.Info("interface name: " + interfaceName)
	log.Info("community name: " + communityName)
	log.Info("local peer IP: " + localPeepIPAddr)
	log.Info("key type: " + keyType)
	log.Info("key file: " + keyFile)
	log.Info("management port: " + managementPort)
	net.CheckOrCreateEdgePortFile()
	// check: if port is given or available.

	if managementPort == "" {
	CutTheLoop:
		for i := 5645; i < 65535; i++ {
			_, err := n.Listen("udp", ":"+string(i))
			if err != nil {
				// then this port is empty.
				managementPort = strconv.Itoa(i)
				break CutTheLoop // cut the loop
			}
		}
	} else if _, err := n.Listen("udp", managementPort); err == nil {
		// ups this is port is being used...
		log.Error("port is used by other process. " + err.Error())
	}
	if managementPort == "" {
		// still??? then there is no port left
		log.Error("no available port in computer")
	}
	net.ProcessEdge(superNodeIPaddr, superNodePort, interfaceName, communityName,
		localPeepIPAddr, keyType, keyFile, managementPort)

}

func listOpenedN2NTunnel() {
	net.PrintN2NTunnels()
}

func removeN2NTunnel(interfaceName, communityName string) {
	pid := net.ReturnPID(interfaceName, communityName)
	i, _ := strconv.Atoi(pid)
	newconf := ""
	log.Check(log.FatalLevel, "remove n2n tunnel: ", syscall.Kill(i, syscall.SIGHUP))

	file, err := os.Open(config.Agent.DataPrefix + "/var/subutai-network/p2p.txt")
	log.Check(log.FatalLevel, "Opening p2p.txt", err)
	scanner := bufio.NewScanner(bufio.NewReader(file))

	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, interfaceName) && strings.HasSuffix(line, communityName) {
			newconf = newconf + line + "\n"
		}
	}
	file.Close()
	log.Check(log.FatalLevel, "Removing p2p tunnel", ioutil.WriteFile(config.Agent.DataPrefix+"/var/subutai-network/p2p.txt", []byte(newconf), 0644))
}

func createGateway(vlanip, vlanid string) {
	net.CreateGateway(vlanip, vlanid)
	log.Info("gateway created")
}

func deleteGateway(vlanid string) {
	net.DeleteGateway(vlanid)
	log.Info("gateway " + vlanid + " removed")
}

func listOpenedTapDevice() {
	net.ListTapDevice()
}

func removeTapDevice(interfaceName string) {
	pid := net.ReturnPID(interfaceName, " ") // i don't know if this will work..
	i, _ := strconv.Atoi(pid)
	log.Check(log.FatalLevel, "remove tap device: ", syscall.Kill(i, syscall.SIGHUP))
	log.Info(interfaceName + " tap device removed")
}

func createVNIMap(tunnelPortName, vni, vlan, envid string) {
	// check: if there is vni file
	if _, err := os.Stat(config.Agent.DataPrefix + "/var/subutai-network/vni_reserve"); os.IsNotExist(err) {
		log.Error("Do Reserve first. No reserved VNIs, not exist file for reserved VNI")
	}
	net.CreateVNIFile(tunnelPortName + "_vni_vlan")
	// check: control if there is such entry in nvi_reserv file.
	ret, _ := net.CheckVNIFile(tunnelPortName+"_vni_vlan", vni, vlan, envid)
	if ret[0] == true {
		log.Info("vni found")
	}
	if ret[1] == true {
		log.Info("vlanid found")
	}
	if ret[2] == true {
		log.Info("envid found")
	}
	if ret[3] == true {
		log.Info("reservation found")
	}

	net.MakeVNIMap(tunnelPortName, vni, vlan, envid)
	log.Info("vni map created: " + vni + " " + vlan + " " + envid)
}

func listVNIMap() {
	if _, err := os.Stat(config.Agent.DataPrefix + "/var/subutai-network/"); os.IsNotExist(err) {
		log.Error("folder not found" + err.Error())
	}
	net.DisplayVNIMap()
}

func delVNI(tunnelPortName, vni, vlan string) {
	if _, err := os.Stat(config.Agent.DataPrefix + "/var/subutai-network/"); os.IsNotExist(err) {
		log.Error("folder not found" + err.Error())
	}
	net.DelVNI(tunnelPortName, vni, vlan)
	log.Info(vni + " " + vlan + " deleted from " + tunnelPortName)
}

func reservVNI(vni, vlan, envid string) {
	// check: create vni file
	net.CreateVNIFile("vni_reserve")
	net.MakeReservation(vni, vlan, envid)
	log.Info(vni + " " + vlan + " " + envid + " is reserved")

}

func reloadN2N(interfaceName, communityName string) {
	net.CheckOrCreateEdgePortFile()
	net.ReloadN2N(interfaceName, communityName)
	log.Info(interfaceName + " " + communityName + " reloaded")
}
<|endoftext|>


<commit_before>// Copyright 2011 Julian Phillips.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package py

// #include "utils.h"
import "C"

import (
	"runtime"
)

type GilState struct {
	state C.PyGILState_STATE
	set   bool
}

func GilState_Ensure() *GilState {
	ret := C.PyGILState_Ensure()
	return &GilState{ret, true}
}

func (g *GilState) Release() {
	if g.set {
		C.PyGILState_Release(g.state)
		g.set = false
	}
}

// Lock is a high-level representation of the Python Global Interpreter Lock
// (GIL) and thread state.  When calling from Go into Python the GIL needs to be
// held and the appropriate thread state loaded.  Lock also calls
// runtime.LockOSThread() to make sure that the calling goroutine doesn't move
// thread whilst calling Python code which would invalidate the per-thread
// state.
//
// Basic usage is:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
// If it appropriate to let other Python threads run (e.g. during a long
// computation, or blocking operation), then there are two options.  Either
// unlock:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
//   // Slow or blocking Go operation
//
//   lock.Lock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
// or unblock threads, which will not call runtme.UnlockOSThread() but it less
// expensive, as we do not free and then recreate a thread state variable:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.UnblockThreads()
//
//   // Slow or blocking Go operation
//
//   lock.BlockThreads()
//
//   // Call Python code ...
//
//   lock.Unlock()
type Lock struct {
	gilState *GilState
	thState  *C.PyThreadState
}

// NewLock returns a new locked Lock
func NewLock() (lock Lock) {
	lock.Lock()
	return
}

// Lock locks the lock.  When it returns everything is setup for calling into
// Python.  No other Python threads will run until either Unlock() or
// UnblockThreads() are called.
//
// If the lock is already locked when this function is called, then nothing
// happens, and the function will return immediately.
func (lock *Lock) Lock() {
	if lock.gilState != nil {
		return
	}

	if lock.thState != nil {
		panic("Lock() called with threads unblocked!")
	}

	runtime.LockOSThread()
	lock.gilState = GilState_Ensure()
}

// Unlock unlocks the lock.  When it returns no calls into Python made be made.
//
// If the lock is not locked when this function is called, then nothing happens,
// and the function returns immediately.  Also, it is not necessay to call
// BlockThreads() before calling Unlock(), even if UnblockThreads() has been
// called.
func (lock *Lock) Unlock() {
	if lock.gilState == nil {
		return
	}

	if lock.thState != nil {
		C.PyEval_RestoreThread(lock.thState)
		lock.thState = nil
	}

	lock.gilState.Release()
	runtime.UnlockOSThread()
	lock.gilState = nil
}

// UnblockThreads() releases the GIL so that other Python threads may run.  It
// does not free the per-thread state created by Lock, nor does it call
// runtime.UnlockOSThread().  This function is intended to allow other Python
// threads to run whilst the calling code is either performing a slow/long
// running operation or is going to block.
//
// Nothing happens if this function is called more than once, all calls but the
// first will be ignored.
func (lock *Lock) UnblockThreads() {
	if lock.gilState == nil {
		panic("UnblockThreads() called on Unlocked Lock")
	}

	if lock.thState == nil {
		lock.thState = C.PyEval_SaveThread()
	}
}

// BlockThreads() reclaims the GIL (and restores per-thread state), after is has
// been released by UnblockThreads().
//
// If this function is called without UnblockThreads() having been called, then
// nothing happens and the function returns immediately.
func (lock *Lock) BlockThreads() {
	if lock.gilState == nil {
		panic("BlockThreads() called on Unlocked Lock")
	}

	if lock.thState != nil {
		C.PyEval_RestoreThread(lock.thState)
		lock.thState = nil
	}
}
<commit_msg>NewLock should return *Lock, not Lock<commit_after>// Copyright 2011 Julian Phillips.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package py

// #include "utils.h"
import "C"

import (
	"runtime"
)

type GilState struct {
	state C.PyGILState_STATE
	set   bool
}

func GilState_Ensure() *GilState {
	ret := C.PyGILState_Ensure()
	return &GilState{ret, true}
}

func (g *GilState) Release() {
	if g.set {
		C.PyGILState_Release(g.state)
		g.set = false
	}
}

// Lock is a high-level representation of the Python Global Interpreter Lock
// (GIL) and thread state.  When calling from Go into Python the GIL needs to be
// held and the appropriate thread state loaded.  Lock also calls
// runtime.LockOSThread() to make sure that the calling goroutine doesn't move
// thread whilst calling Python code which would invalidate the per-thread
// state.
//
// Basic usage is:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
// If it appropriate to let other Python threads run (e.g. during a long
// computation, or blocking operation), then there are two options.  Either
// unlock:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
//   // Slow or blocking Go operation
//
//   lock.Lock()
//
//   // Call Python code ...
//
//   lock.Unlock()
//
// or unblock threads, which will not call runtme.UnlockOSThread() but it less
// expensive, as we do not free and then recreate a thread state variable:
//
//   lock = py.NewLock()
//
//   // Call Python code ...
//
//   lock.UnblockThreads()
//
//   // Slow or blocking Go operation
//
//   lock.BlockThreads()
//
//   // Call Python code ...
//
//   lock.Unlock()
type Lock struct {
	gilState *GilState
	thState  *C.PyThreadState
}

// NewLock returns a new locked Lock
func NewLock() (lock *Lock) {
	lock = new(Lock)
	lock.Lock()
	return
}

// Lock locks the lock.  When it returns everything is setup for calling into
// Python.  No other Python threads will run until either Unlock() or
// UnblockThreads() are called.
//
// If the lock is already locked when this function is called, then nothing
// happens, and the function will return immediately.
func (lock *Lock) Lock() {
	if lock.gilState != nil {
		return
	}

	if lock.thState != nil {
		panic("Lock() called with threads unblocked!")
	}

	runtime.LockOSThread()
	lock.gilState = GilState_Ensure()
}

// Unlock unlocks the lock.  When it returns no calls into Python made be made.
//
// If the lock is not locked when this function is called, then nothing happens,
// and the function returns immediately.  Also, it is not necessay to call
// BlockThreads() before calling Unlock(), even if UnblockThreads() has been
// called.
func (lock *Lock) Unlock() {
	if lock.gilState == nil {
		return
	}

	if lock.thState != nil {
		C.PyEval_RestoreThread(lock.thState)
		lock.thState = nil
	}

	lock.gilState.Release()
	runtime.UnlockOSThread()
	lock.gilState = nil
}

// UnblockThreads() releases the GIL so that other Python threads may run.  It
// does not free the per-thread state created by Lock, nor does it call
// runtime.UnlockOSThread().  This function is intended to allow other Python
// threads to run whilst the calling code is either performing a slow/long
// running operation or is going to block.
//
// Nothing happens if this function is called more than once, all calls but the
// first will be ignored.
func (lock *Lock) UnblockThreads() {
	if lock.gilState == nil {
		panic("UnblockThreads() called on Unlocked Lock")
	}

	if lock.thState == nil {
		lock.thState = C.PyEval_SaveThread()
	}
}

// BlockThreads() reclaims the GIL (and restores per-thread state), after is has
// been released by UnblockThreads().
//
// If this function is called without UnblockThreads() having been called, then
// nothing happens and the function returns immediately.
func (lock *Lock) BlockThreads() {
	if lock.gilState == nil {
		panic("BlockThreads() called on Unlocked Lock")
	}

	if lock.thState != nil {
		C.PyEval_RestoreThread(lock.thState)
		lock.thState = nil
	}
}
<|endoftext|>