Merge pull request extrame#50 from sergeilem/master

FIX: data corruption while reading, issues extrame#31 extrame#46 extrame#47

Author: extrame

.travis.yml

@@ -0,0 +1,23 @@
+language: go
+
+# Force-enable Go modules. This will be unnecessary when Go 1.12 lands.
+env:
+  - GO111MODULE=on
+
+go:
+  - 1.11.x
+
+# Only clone the most recent commit.
+git:
+  depth: 1
+
+# Skip the install step. Don't `go get` dependencies. Only build with the code
+# in vendor/
+install: true
+
+# Don't email me the results of the test runs.
+notifications:
+  email: false
+
+script:
+  - go test -v -race ./...  # Run all the tests with the race detector enabled

BigTable.xls

@@ -2,23 +2,14 @@
 
 [![GoDoc](https://godoc.org/github.com/extrame/xls?status.svg)](https://godoc.org/github.com/extrame/xls)
 
-Pure Golang xls library writen by [Rongshu Tech(chinese)](http://www.rongshu.tech). 
+Pure Golang xls library writen by [Rongshu Tech (chinese)](http://www.rongshu.tech), based on libxls. 
 
-Thanks for contributions from Tamás Gulácsi, sergeilem.
-
-**English User please mailto** [Liu Ming](mailto:liuming@rongshu.tech)
-
-This is a xls library writen in pure Golang. Almostly it is translated from the libxls library in c.
-
-The master brunch has just the reading function without the format. 
-
-***new_formater** branch is for better format for date and number ,but just under test, you can try it in development environment. If you have some problem about the output format, tell me the problem, I will try to fix it.*
+Thanks for contributions from Tamás Gulácsi @tgulacsi, @flyin9.
 
 # Basic Usage
 
 * Use **Open** function for open file
 * Use **OpenWithCloser** function for open file and use the return value closer for close file
 * Use **OpenReader** function for open xls from a reader, you should close related file in your own code
 
-* Follow the example in GODOC
-
+* Follow the example in GoDoc

README.md

@@ -0,0 +1,59 @@
+package xls
+
+import (
+	"fmt"
+	"testing"
+	"time"
+)
+
+func TestBigTable(t *testing.T) {
+	xlFile, err := Open("BigTable.xls", "utf-8")
+	if err != nil {
+		t.Fatalf("Cant open xls file: %s", err)
+	}
+
+	sheet := xlFile.GetSheet(0)
+	if sheet == nil {
+		t.Fatal("Cant get sheet")
+	}
+
+	cnt1 := 1
+	cnt2 := 10000
+	cnt3 := 20000
+	date1, _ := time.Parse("2006-01-02", "2015-01-01")
+	date2, _ := time.Parse("2006-01-02", "2016-01-01")
+	date3, _ := time.Parse("2006-01-02", "2017-01-01")
+
+	for i := 1; i <= 4999; i++ {
+		row := sheet.Row(i)
+		if row == nil {
+			continue
+		}
+
+		col2sample := fmt.Sprintf("%d от %s", cnt1, date1.Format("02.01.2006"))
+		col5sample := fmt.Sprintf("%d от %s", cnt2, date2.Format("02.01.2006"))
+		col8sample := fmt.Sprintf("%d от %s", cnt3, date3.Format("02.01.2006"))
+
+		col2 := row.Col(2)
+		col5 := row.Col(5)
+		col8 := row.Col(8)
+
+		if col2 != col2sample {
+			t.Fatalf("Row %d: col 2 val not eq base value: %s != %s", i, col2, col2sample)
+		}
+		if col5 != col5sample {
+			t.Fatalf("Row %d: col 5 val not eq base value: %s != %s", i, col5, col5sample)
+		}
+		if col8 != col8sample {
+			t.Fatalf("Row %d: col 8 val not eq base value: %s != %s", i, col8, col8sample)
+		}
+
+		cnt1++
+		cnt2++
+		cnt3++
+		date1 = date1.AddDate(0, 0, 1)
+		date2 = date2.AddDate(0, 0, 1)
+		date3 = date3.AddDate(0, 0, 1)
+
+	}
+}

bigtable_test.go

@@ -54,7 +54,16 @@ func (xf *XfRk) String(wb *WorkBook) string {
 		fNo := wb.Xfs[idx].formatNo()
 		if fNo >= 164 { // user defined format
 			if formatter := wb.Formats[fNo]; formatter != nil {
-				if strings.Contains(formatter.str, "#") || strings.Contains(formatter.str, ".00") {
+				formatterLower := strings.ToLower(formatter.str)
+				if formatterLower == "general" ||
+					strings.Contains(formatter.str, "#") ||
+					strings.Contains(formatter.str, ".00") ||
+					strings.Contains(formatterLower, "m/y") ||
+					strings.Contains(formatterLower, "d/y") ||
+					strings.Contains(formatterLower, "m.y") ||
+					strings.Contains(formatterLower, "d.y") ||
+					strings.Contains(formatterLower, "h:") ||
+					strings.Contains(formatterLower, "д.г") {
 					//If format contains # or .00 then this is a number
 					return xf.Rk.String()
 				} else {
@@ -84,7 +93,7 @@ type RK uint32
 func (rk RK) number() (intNum int64, floatNum float64, isFloat bool) {
 	multiplied := rk & 1
 	isInt := rk & 2
-	val := rk >> 2
+	val := int32(rk) >> 2
 	if isInt == 0 {
 		isFloat = true
 		floatNum = math.Float64frombits(uint64(val) << 34)
@@ -93,13 +102,11 @@ func (rk RK) number() (intNum int64, floatNum float64, isFloat bool) {
 		}
 		return
 	}
-	//+++ add lines from here
 	if multiplied != 0 {
 		isFloat = true
 		floatNum = float64(val) / 100
 		return
 	}
-	//+++end
 	return int64(val), 0, false
 }
 

col.go

@@ -0,0 +1,55 @@
+package xls
+
+import (
+	"fmt"
+	"github.com/tealeg/xlsx"
+	"math"
+	"strconv"
+)
+
+//Compares xls and xlsx files
+func CompareXlsXlsx(xlsfilepathname string, xlsxfilepathname string) string {
+	xlsFile, err := Open(xlsfilepathname, "utf-8")
+	if err != nil {
+		return fmt.Sprintf("Cant open xls file: %s", err)
+	}
+
+	xlsxFile, err := xlsx.OpenFile(xlsxfilepathname)
+	if err != nil {
+		return fmt.Sprintf("Cant open xlsx file: %s", err)
+	}
+
+	for sheet, xlsxSheet := range xlsxFile.Sheets {
+		xlsSheet := xlsFile.GetSheet(sheet)
+		if xlsSheet == nil {
+			return fmt.Sprintf("Cant get xls sheet")
+		}
+		for row, xlsxRow := range xlsxSheet.Rows {
+			xlsRow := xlsSheet.Row(row)
+			for cell, xlsxCell := range xlsxRow.Cells {
+				xlsxText := xlsxCell.String()
+				xlsText := xlsRow.Col(cell)
+				if xlsText != xlsxText {
+					//try to convert to numbers
+					xlsFloat, xlsErr := strconv.ParseFloat(xlsText, 64)
+					xlsxFloat, xlsxErr := strconv.ParseFloat(xlsxText, 64)
+					//check if numbers have no significant difference
+					if xlsErr == nil && xlsxErr == nil {
+						diff := math.Abs(xlsFloat - xlsxFloat)
+						if diff > 0.0000001 {
+							return fmt.Sprintf("sheet:%d, row/col: %d/%d, xlsx: (%s)[%d], xls: (%s)[%d], numbers difference: %f.",
+								sheet, row, cell, xlsxText, len(xlsxText),
+								xlsText, len(xlsText), diff)
+						}
+					} else {
+						return fmt.Sprintf("sheet:%d, row/col: %d/%d, xlsx: (%s)[%d], xls: (%s)[%d].",
+							sheet, row, cell, xlsxText, len(xlsxText),
+							xlsText, len(xlsText))
+					}
+				}
+			}
+		}
+	}
+
+	return ""
+}

comparexlsxlsx.go

@@ -0,0 +1,30 @@
+package xls
+
+import (
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestIssue47(t *testing.T) {
+	testdatapath := "testdata"
+	files, err := ioutil.ReadDir(testdatapath)
+	if err != nil {
+		t.Fatalf("Cant read testdata directory contents: %s", err)
+	}
+	for _, f := range files {
+		if filepath.Ext(f.Name()) == ".xls" {
+			xlsfilename := f.Name()
+			xlsxfilename := strings.TrimSuffix(xlsfilename, filepath.Ext(xlsfilename)) + ".xlsx"
+			err := CompareXlsXlsx(path.Join(testdatapath, xlsfilename),
+				path.Join(testdatapath, xlsxfilename))
+			if err != "" {
+				t.Fatalf("XLS file %s an XLSX file are not equal: %s", xlsfilename, err)
+			}
+
+		}
+	}
+
+}