merge

main.go (11362B)

---

 package main
 
 import (
 	"fmt"
 	"os"
 	"log"
 	"path/filepath"
 	"strings"
 	"C"
 	"unsafe"
 	"github.com/360EntSecGroup-Skylar/excelize"
 	"github.com/knieriem/odf/ods"
 	"encoding/csv"
 	"github.com/gotk3/gotk3/gtk"
 	//	"github.com/gotk3/gotk3/gdk"
 )
 
 // object that holds every widget
 type view struct {
 	mainBox *gtk.Box
 	datasetListBox *gtk.ListBox
 	addButton *gtk.Button
 	addDatasetDialog *gtk.FileChooserNativeDialog
 	saveDialog *gtk.FileChooserNativeDialog
 	errorLabel *gtk.Label
 	mergeButton *gtk.Button
 }
 
 // gtk boilerplate
 func windowSetup() *gtk.Window {
 
 	os.Setenv("GSETTINGS_SCHEMA_DIR", ".\\share\\glib-2.0\\schemas")
 	// Set up GTK
 	gtk.Init(&os.Args)
 
 	window, err := gtk.WindowNew(gtk.WINDOW_TOPLEVEL)
 	if err != nil {
 		log.Fatal("Unable to create window:", err)
 	}
 	window.SetTitle("Merge Datasets")
 	window.Connect("destroy", func() {
 		gtk.MainQuit()
 	})
 
 	window.SetName("main-window") // for css selector
 
 	return window
 }
 
 // returns an initial view object (a skeleton that still needs the
 // widgets initialized)
 func viewSetup() view {
 
 	var v view
 	var err error
 
 	// main box container
 	v.mainBox, err = gtk.BoxNew(gtk.ORIENTATION_VERTICAL, 5)
 	if err != nil {
 		log.Fatal("Unable to create grid: ", err)
 	}
 
 	// listBox of dataset names + remove buttons
 	v.datasetListBox, err = gtk.ListBoxNew()
 	v.mainBox.PackStart(v.datasetListBox,false, false, 0)
 
 	// button to add new datasets
 	v.addButton, err = gtk.ButtonNewWithLabel("Add Dataset(s)")
 	if err != nil {
 		log.Fatal("unable to create addButton")
 	}
 	v.mainBox.PackStart(v.addButton, false, false, 0)
 
 
 	// merge datasets
 	v.mergeButton, err = gtk.ButtonNewWithLabel("Merge Datasets")
 	if err != nil {
 		log.Fatal("Unable to create mergeButton", err)
 	}
 	v.mainBox.PackStart(v.mergeButton, false, false, 0)
 
 	v.errorLabel = newLabel("")
 	v.mainBox.PackStart(v.errorLabel, false, false, 0)
 
 	return v
 }
 
 // connect the "Add Dataset" button to the dialog box and the update
 // the list of datasets
 func addButtonSetup(v view, filepaths *[]string, window *gtk.Window) {
 
 	var err error
 
 	// native file chooser
 	v.addDatasetDialog, err = gtk.FileChooserNativeDialogNew("open",window,gtk.FILE_CHOOSER_ACTION_OPEN,"open","cancel")
 	if err != nil {
 		log.Fatal("Can't make addDatasetDialog: ", err)
 	}
 	// user can add multiple datasets
 	v.addDatasetDialog.SetSelectMultiple(true)
 
 	// filters for acceptable filetypes to open
 	CsvFilter, err := gtk.FileFilterNew(); CsvFilter.AddPattern("*.csv"); CsvFilter.SetName("Comma Separated Values")
 	ExcelFilter, err := gtk.FileFilterNew(); ExcelFilter.AddPattern("*.xlsx"); ExcelFilter.SetName("Microsoft Excel")
 	OdsFilter, err := gtk.FileFilterNew(); OdsFilter.AddPattern("*.ods"); OdsFilter.SetName("Open Document Spreadsheet")
 	AllFilter, err := gtk.FileFilterNew(); AllFilter.AddPattern("*"); AllFilter.SetName("All File Types")
 	SpreadsheetFilter, err := gtk.FileFilterNew();
 	SpreadsheetFilter.AddPattern("*.csv");
 	SpreadsheetFilter.AddPattern("*.ods");
 	SpreadsheetFilter.AddPattern("*.xlsx");
 	SpreadsheetFilter.SetName("All Spreadsheets")
 
 	v.addDatasetDialog.AddFilter(CsvFilter)
 	v.addDatasetDialog.AddFilter(ExcelFilter)
 	v.addDatasetDialog.AddFilter(OdsFilter)
 	v.addDatasetDialog.AddFilter(AllFilter)
 	v.addDatasetDialog.AddFilter(SpreadsheetFilter)
 	v.addDatasetDialog.SetFilter(SpreadsheetFilter)
 
 	_, err = v.addButton.Connect("clicked", func() {
 		var response gtk.ResponseType
 		response = gtk.ResponseType(v.addDatasetDialog.Run())
 		list, err := v.addDatasetDialog.GetFilenames()
 		if err == nil && response == gtk.RESPONSE_ACCEPT {
 			// list is a *glib.SList returned by GetFilenames.
 			// glib.SList.Foreach iterates over items in a list
 			// and provides unsafe.Pointers to the C data. Here we
 			// can convert the C []chars to Golang strings using
 			// cgo
 			list.Foreach(func(ptr unsafe.Pointer) {
 				filename := C.GoString((*C.char)(ptr))
 				// add path to list
 				*filepaths = append(*filepaths, filename)
 			})
 			// refresh the view
 			rebuildDatasetListBox(v.datasetListBox, filepaths, window)
 			window.ShowAll()
 		}
 	})
 }
 
 // connect the "Merge Datasets" button to the correct output file
 // chooser and the logic for merging and exporting the datasets
 func mergeButtonSetup(v view, filenames *[]string, window *gtk.Window) {
 	var err error
 
 	// another native file chooser
 	v.saveDialog, err = gtk.FileChooserNativeDialogNew("save",window,gtk.FILE_CHOOSER_ACTION_SAVE,"save","cancel")
 	if err != nil {
 		log.Fatal("Can't make saveDialog: ", err)
 	}
 
 	// unfortunately gtk doesn't seem to provide a way to
 	// enforce file extensions when creating a file with a
 	// gtkfilechooser, so we have to do it manually. These
 	// filters just amount to helpful hints for the user
 	CsvFilter, err := gtk.FileFilterNew(); CsvFilter.AddPattern("*.csv"); CsvFilter.SetName("Comma Separated Values")
 	ExcelFilter, err := gtk.FileFilterNew(); ExcelFilter.AddPattern("*.xlsx"); ExcelFilter.SetName("Microsoft Excel")
 	v.saveDialog.AddFilter(CsvFilter)
 	v.saveDialog.AddFilter(ExcelFilter)
 	v.saveDialog.SetFilter(CsvFilter)
 	v.saveDialog.SetFilename("merged.csv")
 	v.saveDialog.SetDoOverwriteConfirmation(true) // kinda important
 
 	_, err = v.mergeButton.Connect("clicked", func() {
 		// clear old error
 		v.errorLabel.SetText("")
 
 		if goodFileExtensions(filenames) && len(*filenames) != 0 {
 			response := gtk.ResponseType(v.saveDialog.Run())
 
 			if response == gtk.RESPONSE_ACCEPT {
 				outputFile := v.saveDialog.GetFilename()
 
 				// supported file extensions and their associated function for pulling the raw data
 				fileFormat := map[string]func(string) [][]string{
 					".xlsx": pullExcel,
 					".csv": pullCSV,
 					".ods": pullODS,
 				}
 
 				// map of basenames to the [][]string data
 				raws := make(map[string][][]string)
 
 				// all datasets
 				datasets := make(map[string]*Dataset)
 
 				// import dataset based on file extension
 				for _, path := range *filenames {
 					basename := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
 					ext := strings.ToLower(filepath.Ext(path))
 					if _, ok := fileFormat[ext]; ok {
 						raws[basename] = fileFormat[ext](path)
 					}
 					for name, data := range raws {
 						datasets[name] = ImportDataset(name, data)
 					}
 				}
 
 				// merge them into a single Dataset
 				var dataset Dataset
 				dataset.height = 1 // row of terms, even though it's empty
 				dataset.data = make(map[string][]string)
 				for _, d := range datasets {
 					mergeDatasets(&dataset,d)
 				}
 
 				// export dataset
 				exportDataset(&dataset, outputFile)
 			}
 		} else {
 			v.errorLabel.SetText("There was a problem with the files you chose. Make sure to choose supported spreadsheet formats.")
 		}
 	})
 }
 
 // rebuild the list of selected dataset files (with "delete" buttons)
 // and refresh the window
 func rebuildDatasetListBox(list *gtk.ListBox, filenames *[]string, window *gtk.Window) {
 	// clear list
 	list.GetChildren().Foreach(func(item interface{}) {
 		list.Remove(item.(*gtk.Widget))
 	})
 
 	for _, filename := range *filenames {
 		// local variable necessary; see https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
 		path := filename
 
 		// box to hold the two elements
 		box, err := gtk.BoxNew(gtk.ORIENTATION_HORIZONTAL, 2)
 
 		// show the filename
 		text := newLabel(filename)
 		text.SetSelectable(true)
 		box.PackStart(text, false, false, 0)
 
 		// add the "Remove" button
 		removeButton, err := gtk.ButtonNewWithLabel("Remove")
 		if err != nil {
 			log.Fatal("Unable to create removeButton", err)
 		}
 		box.PackEnd(removeButton,false, false, 0)
 		trashImage, err := gtk.ImageNewFromFile("trash.png")
 		removeButton.SetImage(trashImage)
 
 		// "Remove button" removes the path from the list of filepaths and runs this function again
 		_, err = removeButton.Connect("clicked", func() {
 			*filenames = Remove(*filenames, path)
 			rebuildDatasetListBox(list, filenames, window)
 			window.ShowAll()
 		})
 
 		// add the two elements to the overall list
 		list.Insert(box, 0)
 	}
 }
 
 func newLabel(s string) *gtk.Label {
 	l, err := gtk.LabelNew(s)
 	if err != nil {
 		log.Fatal("Unable to create label",s,":", err)
 	}
 	return l
 }
 
 // very dumb file extension checking
 func goodFileExtensions(filepaths *[]string) bool {
 	for _, path := range *filepaths {
 		ext := strings.ToLower(filepath.Ext(path))
 		if ext != ".csv" && ext != ".xlsx" && ext != ".ods" {
 			return false
 		}
 	}
 	return true
 }
 
 func pullExcel(path string) [][]string {
 	var rows [][]string
 	f, err := excelize.OpenFile(path)
 	if err != nil {
 		fmt.Println("Problem importing ", path, err)
 		return rows
 	}
 
 	sheetName := f.GetSheetMap()[1]
 	// fmt.Println(sheetName)
 
 	rows, err = f.GetRows(sheetName)
 	if err != nil {
 		fmt.Println("Problem getting rows from ", path, err)
 	}
 	return rows
 }
 
 func pullCSV(path string) [][]string {
 	var rows [][]string
 
 	f, err := os.Open(path)
 	if err != nil {
 		fmt.Printf("Cannot open '%s': %s\n", path, err.Error())
 		return rows
 	}
 	defer f.Close()
 
 	// TODO do I need to close the reader?
 	r := csv.NewReader(f)
 	r.LazyQuotes = true
 	rows, err = r.ReadAll()
 
 	if err != nil {
 		fmt.Println("trouble reading rows in ", path)
 		return rows
 	}
 	return rows
 }
 
 func pullODS(path string) [][]string {
 	var rows [][]string
 
 	f, err := ods.Open(path)
 	if err != nil {
 		fmt.Println("trouble reading ods file", path,":",err)
 		return rows
 	}
 
 	var d ods.Doc
 
 	err = f.ParseContent(&d)
 
 	if err != nil {
 		fmt.Println("trouble parsing ods file", path,":",err)
 		return rows
 	}
 
 	rows = d.Table[0].Strings()
 	return rows
 }
 
 // see import.go for Dataset fields and methods used in this file
 func mergeDatasets(base *Dataset, new *Dataset) {
 	numOld := base.height - 1
 	numNew := new.height - 1 // number of specimens being added
 
 	// on the first merge the base dataset is empty (no terms or data)
 	if !Include(base.terms, "DATASET") {
 		base.terms = append(base.terms, "DATASET")
 	}
 
 	// add DATASET term to show which file the specimen comes from
 	new.terms = append(new.terms, "DATASET")
 	new.data["DATASET"] = make([]string, numNew)
 	for i, _ := range new.data["DATASET"] {
 		new.data["DATASET"][i] = new.name
 	}
 
 	// add blank data for new terms not present in the old dataset
 	for _, term := range new.terms {
 		if !Include(base.terms, term) {
 			base.terms = append(base.terms, term)
 			base.data[term] = make([]string, numOld)
 		}
 	}
 
 	// add blank data for old terms not present in the new dataset
 	for _, term := range base.terms {
 		if !Include(new.terms, term) {
 			new.terms = append(new.terms, term)
 			new.data[term] = make([]string,  numNew)
 		}
 	}
 
 	// datasets are equalized now (no new data, just blank terms
 	// where appropriate) so merging them is a simple append
 	for _, term := range base.terms {
 		base.data[term] = append(base.data[term], new.data[term]...)
 	}
 
 	// update height
 	base.height += numNew
 }
 
 func main() {
 
 	window := windowSetup()
 
 	// struct containing the main widgets
 	v := viewSetup()
 
 	// holds the paths to the datasets to be merged; this var is
 	// passed around when removing and adding datasets
 	filepaths := &[]string{}
 
 	addButtonSetup(v, filepaths, window)
 	mergeButtonSetup(v, filepaths, window)
 
 	// refresh the (empty) list of datasets for the first time
 	rebuildDatasetListBox(v.datasetListBox, filepaths, window)
 
 	window.Add(v.mainBox)
 
 	window.ShowAll()
 	gtk.Main()
 }