From 2ea67927f5e97161f0dfc7d0b7c2c33aff370d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Tikvi=C4=87?= Date: Wed, 29 Nov 2017 15:46:22 +0100 Subject: [PATCH] added support for metadata update --- http_utility.go | 2 +- json_utility.go | 296 ++++++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 214 insertions(+), 84 deletions(-) diff --git a/http_utility.go b/http_utility.go index 99aa827..f020d9d 100644 --- a/http_utility.go +++ b/http_utility.go @@ -8,7 +8,7 @@ import ( const ( templateHttpErr500_EN = "An internal server error has occurred." templateHttpErr500_RS = "Došlo je do greške na serveru." - templateHttpErr400_EN = "Bad request: invalid request body." + templateHttpErr400_EN = "Bad request." templateHttpErr400_RS = "Neispravan zahtev." templateHttpErr404_EN = "Resource not found." templateHttpErr404_RS = "Objekat nije pronadjen." diff --git a/json_utility.go b/json_utility.go index 80a733a..4209e08 100644 --- a/json_utility.go +++ b/json_utility.go @@ -6,13 +6,22 @@ import ( "fmt" "io" "net/http" + "os" "sync" + "time" "gopkg.in/rana/ora.v4" ) -var mu = &sync.Mutex{} -var payloads []Payload +var ( + mu = &sync.Mutex{} + metadata map[string]Payload + + metadataDB *ora.Ses + activeProject string + + inited bool +) type LangMap map[string]map[string]string @@ -35,7 +44,6 @@ type Translation struct { } type Payload struct { - Type string `json:"type"` Method string `json:"method"` Params map[string]string `json:"params"` Lang []Translation `json:"lang"` @@ -47,21 +55,100 @@ type Payload struct { Data interface{} `json:"data"` } -// InitPayloadsMetaData loads all payloads in the payloads variable. -// Returns an error if it fails. -func InitPayloadsMetaData(db *ora.Ses, project string) error { - payloads = make([]Payload, 0) +// LoadPayloadsdetaData loads all payloads' information into 'metadata' variable. +func LoadPayloadsMetadata(db *ora.Ses, project string, hotloading bool, hlPeriod int) error { + metadataDB = db + activeProject = project - jsonbuf, err := fetchJSON(db, project) + mu.Lock() + defer mu.Unlock() + err := initMetadata(project) if err != nil { return err } + if hotloading { + go hotload(hlPeriod) + } + inited = true + + return nil +} + +func UpdateMetadataModels(md map[string][]byte) (upd, add int, err error) { + if !inited { + return 0, 0, errors.New("webutil: metadata not initialized but update was tried.") + } + + forUpdate := make([]string, 0) + forCreate := make([]string, 0) + + for k, _ := range md { + if _, exists := metadata[k]; exists { + forUpdate = append(forUpdate, k) + } else { + forCreate = append(forCreate, k) + } + } + + for _, k := range forUpdate { + fmt.Printf("for update: %s\n", k) + _, err := metadataDB.PrepAndExe(`update entities set + entity_model = :1 + where entity_type = :2`, + string(md[k]), + k) + + if err != nil { + fmt.Printf("webutility: update metadata: prep and exe: %v\n", err) + continue + } + upd++ + } + + for _, k := range forCreate { + fmt.Printf("for add: %s\n", k) + /* + _, err := metadataDB.PrepAndExe(`insert into entities + (projekat, metadata, entity_type, entity_model) + values(:1, :2, :3, :4)`, + activeProject, "", k, string(md[k])) + + if err != nil { + fmt.Printf("webutility: add metadata: prep and exe: %v\n", err) + continue + } + */ + add++ + } + + return upd, add, nil +} + +func GetMetadataForAllEntities() map[string]Payload { + return metadata +} + +func GetMetadataForEntityType(t string) Payload { + return metadata[t] +} +func UpdateMetadata(entityType string, p *Payload) error { + md, err := json.Marshal(p) + if err != nil { + return err + } + fmt.Printf("md: %s %s\n", entityType, string(md)) mu.Lock() defer mu.Unlock() - json.Unmarshal(jsonbuf, &payloads) - if len(payloads) == 0 { - return errors.New("tables config is corrupt") + _, err = metadataDB.PrepAndExe(`update entities set + metadata = :1 + where projekat = :2 + and entity_type = :3`, + string(md), + activeProject, + entityType) + if err != nil { + return err } return nil } @@ -72,100 +159,143 @@ func DecodeJSON(r io.Reader, v interface{}) error { return json.NewDecoder(r).Decode(v) } -// NewPayload returs a payload sceleton for provided table. -func NewPayload(r *http.Request, table string) Payload { - var pload Payload - +// NewPayload returs a payload sceleton for entity described with etype. +func NewPayload(r *http.Request, etype string) Payload { + pload := metadata[etype] pload.Method = r.Method + " " + r.RequestURI - pload.Type = table - if table != "" { - pload.Params = make(map[string]string, 0) - pload.Lang = translations(table) - pload.Fields = fields(table) - pload.IdField = id(table) - pload.Correlations = correlations(table) - } return pload } -// translations returns a slice of translations for a payload/table of ptype type. -func translations(ptype string) []Translation { - var translations []Translation - - for _, pload := range payloads { - if pload.Type == ptype { - for _, t := range pload.Lang { - translations = append(translations, Translation{ - Language: t.Language, - FieldsLabels: t.FieldsLabels, - }) - } - } +func initMetadata(project string) error { + metadataDB.SetCfg(metadataDB.Cfg().SetClob(ora.S)) + stmt, err := metadataDB.Prep(`select + entity_type, + metadata + from entities + where projekat = `+fmt.Sprintf("'%s'", project), + ora.S, + ora.S) + + defer stmt.Close() + if err != nil { + return err } - return translations -} + rset, err := stmt.Qry() + if err != nil { + return err + } -// fields returns a slice of fields for a payload/table of ptype type. -func fields(ptype string) []Field { - var fields []Field + count := 0 + success := 0 + metadata = make(map[string]Payload) + for rset.Next() { + name := rset.Row[0].(string) + load := []byte(rset.Row[1].(string)) - for _, pload := range payloads { - if pload.Type == ptype { - for _, f := range pload.Fields { - fields = append(fields, f) - } + p := Payload{} + err := json.Unmarshal(load, &p) + if err != nil { + fmt.Printf("couldn't init: '%s' metadata\n", name) + } else { + success++ + metadata[name] = p } + count++ } + fmt.Printf("webutility: successfully loaded %d/%d (%.1f%%) entities\n", + success, count, float32(success)/float32(count)*100.0) - return fields + return nil } -// id returns the name of ID field of a payload/table of ptype type. -func id(ptype string) string { - for _, pload := range payloads { - if pload.Type == ptype { - return pload.IdField +func hotload(n int) { + entityScan := make(map[string]int64) + firstCheck := true + for { + time.Sleep(time.Duration(n) * time.Second) + stmt, err := metadataDB.Prep(`select + ora_rowscn, + entity_type + from entities where projekat = `+fmt.Sprintf("'%s'", activeProject), + ora.I64, + ora.S) + if err != nil { + fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err) + time.Sleep(time.Duration(n) * time.Second) + continue } - } - return "" -} -// correlations returns a slice of correlation fields for a payload/table of ptype type. -func correlations(ptype string) []CorrelationField { - var corr []CorrelationField + rset, err := stmt.Qry() + if err != nil { + fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err) + time.Sleep(time.Duration(n) * time.Second) + continue + } - for _, pload := range payloads { - if pload.Type == ptype { - for _, c := range pload.Correlations { - corr = append(corr, c) + var toRefresh []string + for rset.Next() { + scanID := rset.Row[0].(int64) + entity := rset.Row[1].(string) + oldID, ok := entityScan[entity] + if !ok || oldID != scanID { + entityScan[entity] = scanID + toRefresh = append(toRefresh, entity) } } - } + stmt.Close() - return corr -} + if rset.Err() != nil { + fmt.Fprintf(os.Stderr, "hotload rset error: %v\n", rset.Err()) + time.Sleep(time.Duration(n) * time.Second) + continue + } -// fetchJSON returns a byte slice of JSON configuration file from TABLES_CONFIG table. -// Returns an error if it fails. -func fetchJSON(db *ora.Ses, project string) ([]byte, error) { - db.SetCfg(db.Cfg().SetClob(ora.S)) - stmt, err := db.Prep(`SELECT JSON_NCLOB FROM TABLES_CONFIG WHERE PROJEKAT = `+fmt.Sprintf("'%s'", project), ora.S) - defer stmt.Close() - if err != nil { - return nil, err + if len(toRefresh) > 0 && !firstCheck { + mu.Lock() + refreshMetadata(toRefresh) + mu.Unlock() + } + if firstCheck { + firstCheck = false + } } +} - rset, err := stmt.Qry() - if err != nil { - return nil, err - } +func refreshMetadata(entities []string) { + for _, e := range entities { + //fmt.Printf("refreshing %s\n", e) + stmt, err := metadataDB.Prep(`select + metadata + from entities + where projekat = `+fmt.Sprintf("'%s'", activeProject)+ + ` and entity_type = `+fmt.Sprintf("'%s'", e), + ora.S) - var data string - if rset.Next() { - data = rset.Row[0].(string) - } + if err != nil { + fmt.Printf("webutility: refresh: prep: %v\n", err) + stmt.Close() + continue + } - //fmt.Println(data) - return []byte(data), nil + rset, err := stmt.Qry() + if err != nil { + fmt.Printf("webutility: refresh: query: %v\n", err) + stmt.Close() + continue + } + + for rset.Next() { + load := []byte(rset.Row[0].(string)) + p := Payload{} + err := json.Unmarshal(load, &p) + if err != nil { + fmt.Printf("couldn't refresh: '%s' metadata\n", e) + } else { + metadata[e] = p + //fmt.Printf("unmarshaled %s %v\n", e, metadata[e]) + } + } + stmt.Close() + } } -- 1.8.1.2