v0.2.0

Author: softlandia

.gitignore

@@ -1,3 +1,4 @@
 *.zip
 *.7z
-.idea/*
+.idea/*
+tmp*

.vscode/launch.json

@@ -0,0 +1,17 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${fileDirname}",
+            "env": {},
+            "args": []
+        }
+    ]
+}

char_frac.xlsx

@@ -0,0 +1,23 @@
+package cpd
+
+//codePageTable
+
+//return index of rune in code page table
+//return 0 if rune not in code page table
+func (t *codePageTable) containsRune(r rune) int {
+	for j, e := range *t {
+		if r == e.code {
+			return j
+		}
+	}
+	return 0
+}
+
+func (t *codePageTable) isUpper(r rune) bool {
+	for i := 10; i < len(t); i++ {
+		if r == (*t)[i].code {
+			return true
+		}
+	}
+	return false
+}

codePageTable.go

@@ -0,0 +1,134 @@
+// file from "golang.org\x\text\encoding\internal\identifier" (c) golang autors
+// contain identifier of code page
+// IDCodePage implements interface String()
+
+package cpd
+
+import (
+	"fmt"
+	"strings"
+)
+
+//IDCodePage - index of code page
+type IDCodePage uint16
+
+func (i IDCodePage) String() string {
+	return codePageName[i]
+}
+
+//itRuneMatch - return 1 if rune from this code page, 0 else
+type itRuneMatch func(r rune, tbl *codePageTable) int
+
+//runesMatch - return count of entry elements of data to code page
+type runesMatch func(data []byte, tbl *codePageTable) int
+
+type tableElement struct {
+	code  rune //руна которая нас интересует, она присутствует в этой кодовой таблице как буква алфавита
+	count int  //количество вхождений данной руны
+}
+
+//codePageTable - содержит основные (наиболее часто встречаемые) символы алфавита в данной кодировке
+//первые 8 прописные, 2-я восьмёрка заглавные
+type codePageTable [19]tableElement
+
+//MatchRes - итоговый критерий совпадения массива данных с кодовой страницей
+type MatchRes struct {
+	countMatch int
+}
+
+//CodePage - содержит данные по конкретной кодовой странице
+type CodePage struct {
+	id       IDCodePage    //id of code page
+	name     string        //name of code page
+	MatchRes               //count of matching
+	match    runesMatch    //calculate from input data count of entry to codepage
+	table    codePageTable //table of main alfabet rune of this code page, use for calculate frequency
+}
+
+func (o CodePage) String() string {
+	return fmt.Sprintf("id: %s, countMatch: %d", o.id, o.countMatch)
+}
+
+//MatchingRunes - return string with rune/counts
+func (o CodePage) MatchingRunes() string {
+	var sb strings.Builder
+	fmt.Fprint(&sb, "rune/counts: ")
+	for i, e := range o.table {
+		if i != 0 {
+			fmt.Fprintf(&sb, "%x/%d, ", e.code, e.count)
+		}
+	}
+	return sb.String()
+}
+
+//TCodePages - type for store all code page
+type TCodePages []CodePage
+
+//DeepMach -
+func (o *TCodePages) DeepMach(data []byte) IDCodePage {
+	return ASCII
+}
+
+//Match - return IDCodePage
+//simple calculate count entry data runes in standart code page table
+func (o TCodePages) Match(data []byte) (result IDCodePage) {
+	result = ASCII
+	maxCount := 0
+	for i, cp := range o {
+		o[i].countMatch = cp.match(data, &o[i].table)
+		if o[i].countMatch > maxCount {
+			maxCount = o[i].countMatch
+			result = cp.id
+		}
+	}
+	return result
+}
+
+//CodePages - slice of code pages
+var CodePages = TCodePages{
+	{ASCII, "ASCII", MatchRes{0}, runesMatchASCII,
+		codePageTable{{0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}, {0, 0}}},
+	{IBM866, "IBM866", MatchRes{0}, runesMatch866,
+		codePageTable{
+			//first element serves as sign of absence
+			{0, 0},
+			//о          е		   а		  и			 н			т			с		  р			в
+			{0xAE, 0}, {0xA5, 0}, {0xA0, 0}, {0xA8, 0}, {0xAD, 0}, {0xE2, 0}, {0xE1, 0}, {0xE0, 0}, {0xA2, 0},
+			{0x8E, 0}, {0x85, 0}, {0x80, 0}, {0x88, 0}, {0x8D, 0}, {0x92, 0}, {0x91, 0}, {0x90, 0}, {0x82, 0}}},
+	{UTF8, "UTF8", MatchRes{0}, runesMatchUTF8,
+		codePageTable{
+			{0, 0},
+			//о           е				а		    и			 н			  т			   с			р			в
+			{0xD0BE, 0}, {0xD0B5, 0}, {0xD0B0, 0}, {0xD0B8, 0}, {0xD0BD, 0}, {0xD182, 0}, {0xD181, 0}, {0xD180, 0}, {0xD0B2, 0},
+			{0xD09E, 0}, {0xD095, 0}, {0xD090, 0}, {0xD098, 0}, {0xD0AD, 0}, {0xD0A2, 0}, {0xD0A1, 0}, {0xD0A0, 0}, {0xD092, 0}}},
+	{Windows1251, "Windows1251", MatchRes{0}, runesMatch1251,
+		codePageTable{
+			{0, 0},
+			//а		    и		   н		  с			 р			в		   л		  к			в
+			{0xE0, 0}, {0xE8, 0}, {0xED, 0}, {0xF1, 0}, {0xF0, 0}, {0xE2, 0}, {0xEB, 0}, {0xEA, 0}, {0xE2, 0},
+			{0xC0, 0}, {0xC8, 0}, {0xCD, 0}, {0xD1, 0}, {0xD0, 0}, {0xC2, 0}, {0xCB, 0}, {0xCA, 0}, {0xC2, 0}}},
+	{KOI8R, "KOI8R", MatchRes{0}, runesMatchKOI8,
+		codePageTable{
+			//о		    а		   и		  т			 с			в		   л		  к			в
+			{0, 0},
+			{0xCF, 0}, {0xC1, 0}, {0xC9, 0}, {0xD4, 0}, {0xD3, 0}, {0xD7, 0}, {0xCC, 0}, {0xCB, 0}, {0xD7, 0},
+			{0xEF, 0}, {0xE1, 0}, {0xE9, 0}, {0xF4, 0}, {0xF3, 0}, {0xF7, 0}, {0xEC, 0}, {0xEB, 0}, {0xF7, 0}}},
+}
+
+//codePageName - string of code page name
+var codePageName = map[IDCodePage]string{
+	ASCII:              "ASCII",
+	IBM866:             "IBM866",
+	Windows1251:        "Windows1251",
+	UTF8:               "UTF8",
+	UTF16:              "UTF16",
+	UTF16LE:            "UTF16LE",
+	UTF16BE:            "UTF16BE",
+	UTF32:              "UTF32",
+	KOI8R:              "KOI8R",
+	ISO5427Cyrillic:    "ISO5427Cyrillic",
+	ISO51INISCyrillic:  "ISO51INISCyrillic",
+	ISO111ECMACyrillic: "ISO111ECMACyrillic",
+	ISO153GOST1976874:  "ISO153GOST1976874",
+	Unicode:            "Unicode",
+}

code_pages.go

@@ -0,0 +1,111 @@
+package cpd
+
+const (
+	// ASCII is the uint16 identifier with IANA name US-ASCII (MIME: US-ASCII).
+	// ANSI X3.4-1986
+	// Reference: RFC2046
+	ASCII IDCodePage = 3
+
+	// ISO5427Cyrillic is the uint16 identifier with IANA name ISO_5427.
+	// ISO-IR: International Register of Escape Sequences
+	// Note: The current registration authority is IPSJ/ITSCJ, Japan.
+	// Reference: RFC1345
+	ISO5427Cyrillic IDCodePage = 48
+
+	// ISO51INISCyrillic is the uint16 identifier with IANA name INIS-cyrillic.
+	// ISO-IR: International Register of Escape Sequences
+	// Note: The current registration authority is IPSJ/ITSCJ, Japan.
+	// Reference: RFC1345
+	ISO51INISCyrillic IDCodePage = 53
+
+	// ISO111ECMACyrillic is the uint16 identifier with IANA name ECMA-cyrillic.
+	// ISO registry
+	// (formerly ECMA registry )
+	ISO111ECMACyrillic IDCodePage = 77
+
+	// ISO153GOST1976874 is the uint16 identifier with IANA name GOST_19768-74.
+	// ISO-IR: International Register of Escape Sequences
+	// Note: The current registration authority is IPSJ/ITSCJ, Japan.
+	// Reference: RFC1345
+	ISO153GOST1976874 IDCodePage = 94
+
+	// UTF8 is the uint16 identifier with IANA name UTF-8.
+	//
+	// rfc3629
+	// Reference: RFC3629
+	UTF8 IDCodePage = 106
+
+	// Unicode is the uint16 identifier with IANA name ISO-10646-UCS-2.
+	//
+	// the 2-octet Basic Multilingual Plane, aka Unicode
+	// this needs to specify network byte order: the standard
+	// does not specify (it is a 16-bit integer space)
+	Unicode IDCodePage = 1000
+
+	// UnicodeASCII is the uint16 identifier with IANA name ISO-10646-UCS-Basic.
+	//
+	// ASCII subset of Unicode.  Basic Latin = collection 1
+	// See ISO 10646, Appendix A
+	UnicodeASCII IDCodePage = 1002
+
+	// UTF7 is the uint16 identifier with IANA name UTF-7.
+	//
+	// rfc2152
+	// Reference: RFC2152
+	UTF7 IDCodePage = 1012
+
+	// UTF16BE is the uint16 identifier with IANA name UTF-16BE.
+	//
+	// rfc2781
+	// Reference: RFC2781
+	UTF16BE IDCodePage = 1013
+
+	// UTF16LE is the uint16 identifier with IANA name UTF-16LE.
+	//
+	// rfc2781
+	// Reference: RFC2781
+	UTF16LE IDCodePage = 1014
+
+	// UTF16 is the uint16 identifier with IANA name UTF-16.
+	//
+	// rfc2781
+	// Reference: RFC2781
+	UTF16 IDCodePage = 1015
+
+	// UTF32 is the uint16 identifier with IANA name UTF-32.
+	//
+	// https://www.unicode.org/unicode/reports/tr19/
+	UTF32 IDCodePage = 1017
+
+	// UTF32BE is the uint16 identifier with IANA name UTF-32BE.
+	//
+	// https://www.unicode.org/unicode/reports/tr19/
+	UTF32BE IDCodePage = 1018
+
+	// UTF32LE is the uint16 identifier with IANA name UTF-32LE.
+	//
+	// https://www.unicode.org/unicode/reports/tr19/
+	UTF32LE IDCodePage = 1019
+
+	// KOI8R is the uint16 identifier with IANA name KOI8-R (MIME: KOI8-R).
+	//
+	// rfc1489 , based on GOST-19768-74, ISO-6937/8,
+	// INIS-Cyrillic, ISO-5427.
+	// Reference: RFC1489
+	KOI8R IDCodePage = 2084
+
+	// IBM866 is the uint16 identifier with IANA name IBM866.
+	//
+	// IBM NLDG Volume 2 (SE09-8002-03) August 1994
+	IBM866 IDCodePage = 2086
+
+	// Windows1251 is the uint16 identifier with IANA name windows-1251.
+	//
+	// Microsoft http://www.iana.org/assignments/charset-reg/windows-1251
+	Windows1251 IDCodePage = 2251
+
+	// Windows1252 is the uint16 identifier with IANA name windows-1252.
+	//
+	// Microsoft http://www.iana.org/assignments/charset-reg/windows-1252
+	Windows1252 IDCodePage = 2252
+)

code_pages_id.go

@@ -0,0 +1,45 @@
+package cpd
+
+//checkHeader - check buffer for match to utf-8, utf-16le or utf-16be BOM
+func checkHeader(b []byte) (id IDCodePage, res bool) {
+	if bomUTF8(b) {
+		return UTF8, true
+	}
+	if bomUTF16le(b) {
+		return UTF16LE, true
+	}
+	if bomUTF16be(b) {
+		return UTF16BE, true
+	}
+	return ASCII, false
+}
+
+func bomUTF8(b []byte) bool {
+	if len(b) < 3 {
+		return false
+	}
+	return (b[0] == 0xEF) && (b[1] == 0xBB) && (b[2] == 0xBF)
+}
+
+func bomUTF16le(b []byte) bool {
+	if len(b) < 2 {
+		return false
+	}
+	return (b[0] == 0xFF) && (b[1] == 0xFE)
+}
+
+func bomUTF16be(b []byte) bool {
+	if len(b) < 2 {
+		return false
+	}
+	return (b[0] == 0xFE) && (b[1] == 0xFF)
+}
+
+//ASCII block
+func itASCII(r rune, tbl *codePageTable) int {
+	return 0
+}
+
+func runesMatchASCII(b []byte, tbl *codePageTable) int {
+	return 0
+}