From 31a4e13027906b008bb3cb3469d1f09738875288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Tikvi=C4=87?= Date: Tue, 10 Jul 2018 10:26:57 +0200 Subject: [PATCH] started work on pagination --- json.go | 339 --------------------------------------------------------- payload.go | 363 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 363 insertions(+), 339 deletions(-) delete mode 100644 json.go create mode 100644 payload.go diff --git a/json.go b/json.go deleted file mode 100644 index 724536c..0000000 --- a/json.go +++ /dev/null @@ -1,339 +0,0 @@ -package webutility - -import ( - "database/sql" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "sync" - "time" - - "git.to-net.rs/marko.tikvic/gologger" -) - -var ( - mu = &sync.Mutex{} - metadata = make(map[string]Payload) - - updateQue = make(map[string][]byte) - - metadataDB *sql.DB - activeProject string - - inited bool - driver string - logger *gologger.Logger -) - -type LangMap map[string]map[string]string - -type Field struct { - Parameter string `json:"param"` - Type string `json:"type"` - Visible bool `json:"visible"` - Editable bool `json:"editable"` -} - -type CorrelationField struct { - Result string `json:"result"` - Elements []string `json:"elements"` - Type string `json:"type"` -} - -type Translation struct { - Language string `json:"language"` - FieldsLabels map[string]string `json:"fieldsLabels"` -} - -type Payload struct { - Method string `json:"method"` - Params map[string]string `json:"params"` - Lang []Translation `json:"lang"` - Fields []Field `json:"fields"` - Correlations []CorrelationField `json:"correlationFields"` - IdField string `json:"idField"` - - // Data holds JSON payload. - // It can't be used for itteration. - Data interface{} `json:"data"` -} - -func (p *Payload) SetData(data interface{}) { - p.Data = data -} - -// NewPayload returs a payload sceleton for entity described with etype. -func NewPayload(r *http.Request, etype string) Payload { - pload := metadata[etype] - pload.Method = r.Method + " " + r.RequestURI - return pload -} - -// DecodeJSON decodes JSON data from r to v. -// Returns an error if it fails. -func DecodeJSON(r io.Reader, v interface{}) error { - return json.NewDecoder(r).Decode(v) -} - -// InitPayloadsMetadata loads all payloads' information into 'metadata' variable. -func InitPayloadsMetadata(drv string, db *sql.DB, project string) error { - var err error - if drv != "ora" && drv != "mysql" { - err = errors.New("driver not supported") - return err - } - - driver = drv - metadataDB = db - activeProject = project - - logger, err = gologger.New("metadata", gologger.MaxLogSize100KB) - if err != nil { - fmt.Printf("webutility: %s\n", err.Error()) - } - - mu.Lock() - defer mu.Unlock() - err = initMetadata(project) - if err != nil { - return err - } - inited = true - - return nil -} - -func EnableHotloading(interval int) { - if interval > 0 { - go hotload(interval) - } -} - -func GetMetadataForAllEntities() map[string]Payload { - return metadata -} - -func GetMetadataForEntity(t string) (Payload, bool) { - p, ok := metadata[t] - return p, ok -} - -func QueEntityModelUpdate(entityType string, v interface{}) { - updateQue[entityType], _ = json.Marshal(v) -} - -func UpdateEntityModels(command string) (total, upd, add int, err error) { - if command != "force" && command != "missing" { - return total, 0, 0, errors.New("webutility: unknown command: " + command) - } - - if !inited { - return 0, 0, 0, errors.New("webutility: metadata not initialized but update was tried.") - } - - total = len(updateQue) - - toUpdate := make([]string, 0) - toAdd := make([]string, 0) - - for k, _ := range updateQue { - if _, exists := metadata[k]; exists { - if command == "force" { - toUpdate = append(toUpdate, k) - } - } else { - toAdd = append(toAdd, k) - } - } - - var uStmt *sql.Stmt - if driver == "ora" { - uStmt, err = metadataDB.Prepare("update entities set entity_model = :1 where entity_type = :2") - if err != nil { - logger.Trace(err.Error()) - return - } - } else if driver == "mysql" { - uStmt, err = metadataDB.Prepare("update entities set entity_model = ? where entity_type = ?") - if err != nil { - logger.Trace(err.Error()) - return - } - } - for _, k := range toUpdate { - _, err = uStmt.Exec(string(updateQue[k]), k) - if err != nil { - logger.Trace(err.Error()) - return - } - upd++ - } - - blankPayload, _ := json.Marshal(Payload{}) - var iStmt *sql.Stmt - if driver == "ora" { - iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(:1, :2, :3, :4)") - if err != nil { - logger.Trace(err.Error()) - return - } - } else if driver == "mysql" { - iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(?, ?, ?, ?)") - if err != nil { - logger.Trace(err.Error()) - return - } - } - for _, k := range toAdd { - _, err = iStmt.Exec(activeProject, string(blankPayload), k, string(updateQue[k])) - if err != nil { - logger.Trace(err.Error()) - return - } - metadata[k] = Payload{} - add++ - } - - return total, upd, add, nil -} - -func initMetadata(project string) error { - rows, err := metadataDB.Query(`select - entity_type, - metadata - from entities - where projekat = ` + fmt.Sprintf("'%s'", project)) - if err != nil { - return err - } - defer rows.Close() - - if len(metadata) > 0 { - metadata = nil - } - metadata = make(map[string]Payload) - for rows.Next() { - var name, load string - rows.Scan(&name, &load) - - p := Payload{} - err := json.Unmarshal([]byte(load), &p) - if err != nil { - logger.Log("webutility: couldn't init: '%s' metadata: %s:\n%s\n", name, err.Error(), load) - } else { - metadata[name] = p - } - } - - return nil -} - -func hotload(n int) { - entityScan := make(map[string]int64) - firstCheck := true - for { - time.Sleep(time.Duration(n) * time.Second) - rows, err := metadataDB.Query(`select - ora_rowscn, - entity_type - from entities where projekat = ` + fmt.Sprintf("'%s'", activeProject)) - if err != nil { - logger.Log("webutility: hotload failed: %v\n", err) - time.Sleep(time.Duration(n) * time.Second) - continue - } - - var toRefresh []string - for rows.Next() { - var scanID int64 - var entity string - rows.Scan(&scanID, &entity) - oldID, ok := entityScan[entity] - if !ok || oldID != scanID { - entityScan[entity] = scanID - toRefresh = append(toRefresh, entity) - } - } - rows.Close() - - if rows.Err() != nil { - logger.Log("webutility: hotload rset error: %v\n", rows.Err()) - time.Sleep(time.Duration(n) * time.Second) - continue - } - - if len(toRefresh) > 0 && !firstCheck { - mu.Lock() - refreshMetadata(toRefresh) - mu.Unlock() - } - if firstCheck { - firstCheck = false - } - } -} - -func refreshMetadata(entities []string) { - for _, e := range entities { - fmt.Printf("refreshing %s\n", e) - rows, err := metadataDB.Query(`select - metadata - from entities - where projekat = ` + fmt.Sprintf("'%s'", activeProject) + - ` and entity_type = ` + fmt.Sprintf("'%s'", e)) - - if err != nil { - logger.Log("webutility: refresh: prep: %v\n", err) - rows.Close() - continue - } - - for rows.Next() { - var load string - rows.Scan(&load) - p := Payload{} - err := json.Unmarshal([]byte(load), &p) - if err != nil { - logger.Log("webutility: couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), load) - } else { - metadata[e] = p - } - } - rows.Close() - } -} - -/* -func ModifyMetadataForEntity(entityType string, p *Payload) error { - md, err := json.Marshal(*p) - if err != nil { - return err - } - - mu.Lock() - defer mu.Unlock() - _, err = metadataDB.PrepAndExe(`update entities set - metadata = :1 - where projekat = :2 - and entity_type = :3`, - string(md), - activeProject, - entityType) - if err != nil { - return err - } - return nil -} - -func DeleteEntityModel(entityType string) error { - _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType) - if err == nil { - mu.Lock() - delete(metadata, entityType) - mu.Unlock() - } - return err -} -*/ diff --git a/payload.go b/payload.go new file mode 100644 index 0000000..f9c3e40 --- /dev/null +++ b/payload.go @@ -0,0 +1,363 @@ +package webutility + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "sync" + "time" + + "git.to-net.rs/marko.tikvic/gologger" +) + +var ( + mu = &sync.Mutex{} + metadata = make(map[string]Payload) + + updateQue = make(map[string][]byte) + + metadataDB *sql.DB + activeProject string + + inited bool + driver string + logger *gologger.Logger +) + +type LangMap map[string]map[string]string + +type Field struct { + Parameter string `json:"param"` + Type string `json:"type"` + Visible bool `json:"visible"` + Editable bool `json:"editable"` +} + +type CorrelationField struct { + Result string `json:"result"` + Elements []string `json:"elements"` + Type string `json:"type"` +} + +type Translation struct { + Language string `json:"language"` + FieldsLabels map[string]string `json:"fieldsLabels"` +} + +type PaginationLinks struct { + Base string `json:"base"` + Next string `json:"next"` + Prev string `json:"prev"` + Self string `json:"self"` +} + +type PaginationParameters struct { + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` + SortBy string `json:"sortBy"` + Order string `json:"order"` +} + +type Payload struct { + Method string `json:"method"` + Params map[string]string `json:"params"` + Lang []Translation `json:"lang"` + Fields []Field `json:"fields"` + Correlations []CorrelationField `json:"correlationFields"` + IdField string `json:"idField"` + + // Pagination + Count int64 `json:"count"` + Total int64 `json:"total"` + Links *PaginationLinks `json:"_links"` + + // Data holds JSON payload. It can't be used for itteration. + Data interface{} `json:"data"` +} + +func (p *Payload) SetData(data interface{}) { + p.Data = data +} + +func (p *Payload) SetPaginationInfo(reqUrl string, count, total int64, params PaginationParameters) { + p.Count = count + p.Total = total + +} + +// NewPayload returs a payload sceleton for entity described with etype. +func NewPayload(r *http.Request, etype string) Payload { + pload := metadata[etype] + pload.Method = r.Method + " " + r.RequestURI + return pload +} + +// DecodeJSON decodes JSON data from r to v. +// Returns an error if it fails. +func DecodeJSON(r io.Reader, v interface{}) error { + return json.NewDecoder(r).Decode(v) +} + +// InitPayloadsMetadata loads all payloads' information into 'metadata' variable. +func InitPayloadsMetadata(drv string, db *sql.DB, project string) error { + var err error + if drv != "ora" && drv != "mysql" { + err = errors.New("driver not supported") + return err + } + + driver = drv + metadataDB = db + activeProject = project + + logger, err = gologger.New("metadata", gologger.MaxLogSize100KB) + if err != nil { + fmt.Printf("webutility: %s\n", err.Error()) + } + + mu.Lock() + defer mu.Unlock() + err = initMetadata(project) + if err != nil { + return err + } + inited = true + + return nil +} + +func EnableHotloading(interval int) { + if interval > 0 { + go hotload(interval) + } +} + +func GetMetadataForAllEntities() map[string]Payload { + return metadata +} + +func GetMetadataForEntity(t string) (Payload, bool) { + p, ok := metadata[t] + return p, ok +} + +func QueEntityModelUpdate(entityType string, v interface{}) { + updateQue[entityType], _ = json.Marshal(v) +} + +func UpdateEntityModels(command string) (total, upd, add int, err error) { + if command != "force" && command != "missing" { + return total, 0, 0, errors.New("webutility: unknown command: " + command) + } + + if !inited { + return 0, 0, 0, errors.New("webutility: metadata not initialized but update was tried.") + } + + total = len(updateQue) + + toUpdate := make([]string, 0) + toAdd := make([]string, 0) + + for k, _ := range updateQue { + if _, exists := metadata[k]; exists { + if command == "force" { + toUpdate = append(toUpdate, k) + } + } else { + toAdd = append(toAdd, k) + } + } + + var uStmt *sql.Stmt + if driver == "ora" { + uStmt, err = metadataDB.Prepare("update entities set entity_model = :1 where entity_type = :2") + if err != nil { + logger.Trace(err.Error()) + return + } + } else if driver == "mysql" { + uStmt, err = metadataDB.Prepare("update entities set entity_model = ? where entity_type = ?") + if err != nil { + logger.Trace(err.Error()) + return + } + } + for _, k := range toUpdate { + _, err = uStmt.Exec(string(updateQue[k]), k) + if err != nil { + logger.Trace(err.Error()) + return + } + upd++ + } + + blankPayload, _ := json.Marshal(Payload{}) + var iStmt *sql.Stmt + if driver == "ora" { + iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(:1, :2, :3, :4)") + if err != nil { + logger.Trace(err.Error()) + return + } + } else if driver == "mysql" { + iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(?, ?, ?, ?)") + if err != nil { + logger.Trace(err.Error()) + return + } + } + for _, k := range toAdd { + _, err = iStmt.Exec(activeProject, string(blankPayload), k, string(updateQue[k])) + if err != nil { + logger.Trace(err.Error()) + return + } + metadata[k] = Payload{} + add++ + } + + return total, upd, add, nil +} + +func initMetadata(project string) error { + rows, err := metadataDB.Query(`select + entity_type, + metadata + from entities + where projekat = ` + fmt.Sprintf("'%s'", project)) + if err != nil { + return err + } + defer rows.Close() + + if len(metadata) > 0 { + metadata = nil + } + metadata = make(map[string]Payload) + for rows.Next() { + var name, load string + rows.Scan(&name, &load) + + p := Payload{} + err := json.Unmarshal([]byte(load), &p) + if err != nil { + logger.Log("webutility: couldn't init: '%s' metadata: %s:\n%s\n", name, err.Error(), load) + } else { + metadata[name] = p + } + } + + return nil +} + +func hotload(n int) { + entityScan := make(map[string]int64) + firstCheck := true + for { + time.Sleep(time.Duration(n) * time.Second) + rows, err := metadataDB.Query(`select + ora_rowscn, + entity_type + from entities where projekat = ` + fmt.Sprintf("'%s'", activeProject)) + if err != nil { + logger.Log("webutility: hotload failed: %v\n", err) + time.Sleep(time.Duration(n) * time.Second) + continue + } + + var toRefresh []string + for rows.Next() { + var scanID int64 + var entity string + rows.Scan(&scanID, &entity) + oldID, ok := entityScan[entity] + if !ok || oldID != scanID { + entityScan[entity] = scanID + toRefresh = append(toRefresh, entity) + } + } + rows.Close() + + if rows.Err() != nil { + logger.Log("webutility: hotload rset error: %v\n", rows.Err()) + time.Sleep(time.Duration(n) * time.Second) + continue + } + + if len(toRefresh) > 0 && !firstCheck { + mu.Lock() + refreshMetadata(toRefresh) + mu.Unlock() + } + if firstCheck { + firstCheck = false + } + } +} + +func refreshMetadata(entities []string) { + for _, e := range entities { + fmt.Printf("refreshing %s\n", e) + rows, err := metadataDB.Query(`select + metadata + from entities + where projekat = ` + fmt.Sprintf("'%s'", activeProject) + + ` and entity_type = ` + fmt.Sprintf("'%s'", e)) + + if err != nil { + logger.Log("webutility: refresh: prep: %v\n", err) + rows.Close() + continue + } + + for rows.Next() { + var load string + rows.Scan(&load) + p := Payload{} + err := json.Unmarshal([]byte(load), &p) + if err != nil { + logger.Log("webutility: couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), load) + } else { + metadata[e] = p + } + } + rows.Close() + } +} + +/* +func ModifyMetadataForEntity(entityType string, p *Payload) error { + md, err := json.Marshal(*p) + if err != nil { + return err + } + + mu.Lock() + defer mu.Unlock() + _, err = metadataDB.PrepAndExe(`update entities set + metadata = :1 + where projekat = :2 + and entity_type = :3`, + string(md), + activeProject, + entityType) + if err != nil { + return err + } + return nil +} + +func DeleteEntityModel(entityType string) error { + _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType) + if err == nil { + mu.Lock() + delete(metadata, entityType) + mu.Unlock() + } + return err +} +*/ -- 1.8.1.2