Compare commits
4 Commits
Author | SHA1 | Date |
---|---|---|
Georg Spar | 4b3b39cd59 | 2 years ago |
Georg Spar | 23bab487aa | 2 years ago |
Georg Spar | c8fb0de2a9 | 2 years ago |
Georg Spar | 44c421983d | 2 years ago |
16 changed files with 318 additions and 310 deletions
@ -1,116 +0,0 @@
|
||||
package docstract |
||||
|
||||
import ( |
||||
"errors" |
||||
"io/ioutil" |
||||
"strings" |
||||
) |
||||
|
||||
//DocType is a wrapper for the type iota/enum
|
||||
type DocType int |
||||
|
||||
const ( |
||||
//DocUnkown represents an unknown document type
|
||||
DocUnkown = iota |
||||
|
||||
//DocPDF represents a pdf document type
|
||||
DocPDF |
||||
|
||||
//DocX represents a microsoft docx document type
|
||||
DocX |
||||
|
||||
//DocXLSX represents microsoft excel doc
|
||||
DocXLSX |
||||
|
||||
//DocHTML represents an html document type
|
||||
DocHTML |
||||
) |
||||
|
||||
//DocStract stores the binary data for extracted files, as well as the type and filename metadata
|
||||
type DocStract struct { |
||||
Type DocType |
||||
FileName *string |
||||
Bytes []byte |
||||
} |
||||
|
||||
//SaveFile saves the file to the path, does not check if it's an unkown filetype only if it has a name
|
||||
func (d *DocStract) SaveFile(path string) error { |
||||
if len(path) > 0 && path[len(path)-1] != '/' { |
||||
path += "/" |
||||
} |
||||
|
||||
if d.FileName == nil { |
||||
return errors.New("document does not have a filename cannot save") |
||||
} |
||||
|
||||
return ioutil.WriteFile(path+*(d.FileName), d.Bytes, 0644) |
||||
} |
||||
|
||||
//sets name to nil if cannot dertermine name and type to unkown
|
||||
func (d *DocStract) getName() { |
||||
blocks := strings.Split(string(d.Bytes), "\n") |
||||
nameBlock := blocks[len(blocks)-1] |
||||
|
||||
chunks := strings.Split(nameBlock, ".") |
||||
|
||||
nameChunk := 0 |
||||
t := DocUnkown |
||||
|
||||
switch len(chunks[0]) { |
||||
case 0: //pdf
|
||||
t = DocPDF |
||||
nameChunk = 2 |
||||
default: //html
|
||||
switch { |
||||
case strings.Contains(chunks[0], "word"): //docx
|
||||
nameChunk = 8 |
||||
t = DocX |
||||
break |
||||
case strings.Contains(chunks[2], "worksheets"): //xlsx
|
||||
t = DocXLSX |
||||
for i := 3; i < len(chunks); i++ { |
||||
if strings.Contains(StripSeperators(chunks[i]), "xlsx") { |
||||
nameChunk = i + 1 |
||||
break |
||||
} |
||||
} |
||||
break |
||||
default: //html
|
||||
t = DocHTML |
||||
} |
||||
} |
||||
|
||||
name := strings.TrimSpace(chunks[nameChunk]) |
||||
name = StripSeperators(name) |
||||
|
||||
switch t { |
||||
case DocPDF: |
||||
name += ".pdf" |
||||
name = name[3:] |
||||
d.Type = DocPDF |
||||
|
||||
case DocX: |
||||
name += ".docx" |
||||
name = name[3:] |
||||
d.Type = DocX |
||||
|
||||
case DocXLSX: |
||||
name += ".xlsx" |
||||
head := 0 |
||||
for i, b := range d.Bytes { |
||||
if i+1 < len(d.Bytes) && b == 'P' && d.Bytes[i+1] == 'K' { |
||||
head = i |
||||
break |
||||
} |
||||
} |
||||
name = name[3:] |
||||
d.Type = DocXLSX |
||||
d.Bytes = d.Bytes[head:] |
||||
|
||||
case DocHTML: |
||||
name += ".html" |
||||
d.Type = DocHTML |
||||
} |
||||
|
||||
d.FileName = &name |
||||
} |
@ -1,10 +0,0 @@
|
||||
module AUDIAG/rtcusertr/DocStract |
||||
|
||||
go 1.18 |
||||
|
||||
require ( |
||||
github.com/pkg/errors v0.9.1 |
||||
github.com/richardlehane/mscfb v1.0.4 |
||||
) |
||||
|
||||
require github.com/richardlehane/msoleps v1.0.1 // indirect |
@ -1,6 +0,0 @@
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= |
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
||||
github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= |
||||
github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= |
||||
github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o= |
||||
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= |
@ -1,20 +0,0 @@
|
||||
package docstract |
||||
|
||||
//StripSeperators removes all the random 0 bytes
|
||||
func StripSeperators(s string) string { |
||||
iBytes := []byte(s) |
||||
oBytes := []byte{} |
||||
|
||||
if len(iBytes) >= 3 { |
||||
offset := 0 |
||||
if iBytes[0] == iBytes[2] && iBytes[0] == byte(0) { |
||||
offset = 1 |
||||
} |
||||
|
||||
for i := offset; i < len(iBytes); i += 2 { |
||||
oBytes = append(oBytes, iBytes[i]) |
||||
} |
||||
} |
||||
|
||||
return string(oBytes) |
||||
} |
@ -1,64 +0,0 @@
|
||||
package docstract |
||||
|
||||
import ( |
||||
"bytes" |
||||
"strings" |
||||
"sync" |
||||
|
||||
"github.com/pkg/errors" |
||||
"github.com/richardlehane/mscfb" |
||||
) |
||||
|
||||
//Extract takes a .msg files binary data and returns an array of attachments and a count of how many files were extracted
|
||||
func Extract(data []byte) (*[]*DocStract, int, error) { |
||||
|
||||
reader := bytes.NewReader(data) |
||||
|
||||
doc, err := mscfb.New(reader) |
||||
|
||||
if err != nil { |
||||
return nil, 0, errors.Wrap(err, "creating reader") |
||||
} |
||||
|
||||
files := []*DocStract{} |
||||
|
||||
{ // Get all the attachments seperated to parse
|
||||
attachment := false |
||||
file := 0 |
||||
for entry, err := doc.Next(); err == nil; entry, err = doc.Next() { |
||||
|
||||
if strings.Contains(entry.Name, "attach") { |
||||
files = append(files, &DocStract{}) |
||||
attachment = true |
||||
continue |
||||
} |
||||
if attachment && strings.Contains(entry.Name, "properties") { |
||||
attachment = false |
||||
file++ |
||||
continue |
||||
} |
||||
|
||||
if attachment { |
||||
buf := make([]byte, entry.Size) |
||||
i, _ := entry.Read(buf) |
||||
if i > 0 { |
||||
files[file].Bytes = append(files[file].Bytes, buf[:i]...) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
{ //Determine FileType and FileName
|
||||
wait := sync.WaitGroup{} |
||||
for _, doc := range files { |
||||
wait.Add(1) |
||||
go func(d *DocStract) { |
||||
d.getName() |
||||
wait.Done() |
||||
}(doc) |
||||
} |
||||
wait.Wait() |
||||
} |
||||
|
||||
return &files, len(files), nil |
||||
} |
@ -0,0 +1,116 @@
|
||||
package database |
||||
|
||||
import ( |
||||
"database/sql" |
||||
"log" |
||||
|
||||
"gitlab.dedyn.io/AUDIAG/rtcusertr/domain" |
||||
|
||||
_ "github.com/go-sql-driver/mysql" |
||||
) |
||||
|
||||
var ( |
||||
dbConnection = "rtcusertrdbuser:hurtz3585@tcp(ubodroid-1:3306)/rtcusertrdb" |
||||
dbType = "mysql" |
||||
) |
||||
|
||||
// Tool Funktionen
|
||||
func CreateKnoten(knoten domain.KnotenE) (resKnoten domain.KnotenE, err error) { |
||||
conn, err := sql.Open(dbType, dbConnection) |
||||
if err != nil { |
||||
log.Println("Error while connecting DB: ", err) |
||||
} |
||||
log.Println("Verbindung hergestellt") |
||||
log.Println("Name = ", knoten.Fachbereich) |
||||
defer conn.Close() |
||||
|
||||
res, err := conn.Exec("INSERT INTO knoten VALUES(?,?,?,?)", knoten.Id, knoten.Fachbereich, knoten.Anzahl, knoten.Tag) |
||||
if err != nil { |
||||
log.Println("Error while executing insert statement", err) |
||||
} |
||||
lastId, err := res.LastInsertId() |
||||
knoten.Id = uint(lastId) |
||||
resKnoten = knoten |
||||
|
||||
return resKnoten, err |
||||
} |
||||
|
||||
func DeleteKnoten(knotenId uint) (result, err error) { |
||||
conn, err := sql.Open(dbType, dbConnection) |
||||
if err != nil { |
||||
log.Println("Error while connecting DB: ", err) |
||||
} |
||||
log.Println("DB Verbindung hergestellt") |
||||
defer conn.Close() |
||||
|
||||
_, err = conn.Exec("DELETE FROM knoten WHERE id = ?", knotenId) |
||||
if err != nil { |
||||
log.Fatal("Error while executing DELETE statement", err) |
||||
} |
||||
return result, err |
||||
} |
||||
|
||||
func ShowKnoten() (knotenArray []domain.KnotenE, err error) { |
||||
conn, err := sql.Open(dbType, dbConnection) |
||||
if err != nil { |
||||
log.Println("Error while connecting DB: ", err) |
||||
} |
||||
log.Println("DB Verbindung hergestellt") |
||||
defer conn.Close() |
||||
|
||||
results, err := conn.Query("SELECT * FROM knoten") |
||||
if err != nil { |
||||
log.Fatal("Error while executing SELECt statement", err) |
||||
} |
||||
|
||||
var knoten domain.KnotenE |
||||
for results.Next() { |
||||
err = results.Scan(&knoten.Id, &knoten.Fachbereich, &knoten.Anzahl, &knoten.Tag) |
||||
if err != nil { |
||||
log.Fatal("Error: ", err) |
||||
} |
||||
knotenArray = append(knotenArray, knoten) |
||||
} |
||||
return knotenArray, err |
||||
} |
||||
|
||||
func GetKnoten(id uint) (knoten domain.KnotenE, err error) { |
||||
conn, err := sql.Open(dbType, dbConnection) |
||||
if err != nil { |
||||
log.Println("Error while connecting DB: ", err) |
||||
} |
||||
log.Println("DB Verbindung hergestellt") |
||||
defer conn.Close() |
||||
|
||||
results, err := conn.Query("SELECT * FROM knoten WHERE id=?", id) |
||||
if err != nil { |
||||
log.Fatal("Error while executing SELECt statement", err) |
||||
} |
||||
for results.Next() { |
||||
err = results.Scan(&knoten.Id, &knoten.Fachbereich, &knoten.Anzahl, &knoten.Tag) |
||||
if err != nil { |
||||
log.Fatal("Error: ", err) |
||||
} |
||||
} |
||||
|
||||
return knoten, err |
||||
} |
||||
|
||||
func UpdateKnoten(knoten domain.KnotenE) (resKnoten domain.KnotenE, err error) { |
||||
conn, err := sql.Open(dbType, dbConnection) |
||||
if err != nil { |
||||
log.Println("Error while connecting DB: ", err) |
||||
} |
||||
log.Println("Verbindung hergestellt") |
||||
log.Println("Name = ", knoten.Fachbereich) |
||||
defer conn.Close() |
||||
|
||||
_, err = conn.Exec("UPDATE knoten SET fachbereich=?, anzahl=?, tag=? WHERE id=?", knoten.Fachbereich, knoten.Anzahl, knoten.Tag, knoten.Id) |
||||
if err != nil { |
||||
log.Println("Error while executing insert statement", err) |
||||
} |
||||
|
||||
resKnoten = knoten |
||||
|
||||
return resKnoten, err |
||||
} |
@ -0,0 +1,5 @@
|
||||
module database.go |
||||
|
||||
go 1.18 |
||||
|
||||
require github.com/go-sql-driver/mysql v1.6.0 // indirect |
@ -0,0 +1,2 @@
|
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= |
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= |
@ -0,0 +1,8 @@
|
||||
package domain |
||||
|
||||
type KnotenE struct { |
||||
Id uint `json:"id"` |
||||
Fachbereich string `json:"fachbereich"` |
||||
Anzahl int `json:"anzahl"` |
||||
Tag string `json:"tag"` |
||||
} |
@ -1,18 +1,19 @@
|
||||
module rtcusertr.go |
||||
module gitlab.dedyn.io/g.spar/rtcusertr |
||||
|
||||
go 1.18 |
||||
|
||||
require ( |
||||
AUDIAG/rtcusertr/DocStract v0.0.0-00010101000000-000000000000 |
||||
github.com/sirupsen/logrus v1.4.2 |
||||
github.com/gorilla/mux v1.8.0 |
||||
github.com/sirupsen/logrus v1.9.0 |
||||
gitlab.dedyn.io/AUDIAG/rtcusertr/database v0.0.0-00010101000000-000000000000 |
||||
gitlab.dedyn.io/AUDIAG/rtcusertr/domain v0.0.0-00010101000000-000000000000 |
||||
) |
||||
|
||||
require ( |
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect |
||||
github.com/pkg/errors v0.9.1 // indirect |
||||
github.com/richardlehane/mscfb v1.0.4 // indirect |
||||
github.com/richardlehane/msoleps v1.0.1 // indirect |
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 // indirect |
||||
github.com/go-sql-driver/mysql v1.6.0 // indirect |
||||
golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect |
||||
) |
||||
|
||||
replace AUDIAG/rtcusertr/DocStract => ./DocStract |
||||
replace gitlab.dedyn.io/AUDIAG/rtcusertr/database => ./database |
||||
|
||||
replace gitlab.dedyn.io/AUDIAG/rtcusertr/domain => ./domain |
||||
|
@ -1,19 +1,20 @@
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= |
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= |
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= |
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= |
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= |
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= |
||||
github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= |
||||
github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= |
||||
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= |
||||
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= |
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= |
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= |
||||
github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= |
||||
github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= |
||||
github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o= |
||||
github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= |
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= |
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= |
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= |
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= |
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= |
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= |
||||
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= |
||||
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= |
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= |
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= |
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= |
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
||||
golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8= |
||||
golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= |
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= |
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= |
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= |
||||
|
Binary file not shown.
Loading…
Reference in new issue