Marko Tikvić / webutility

Download as
Browse Code »

Commit 2ea67927f5e97161f0dfc7d0b7c2c33aff370d2a

Authored by Marko Tikvić
1 parent 6620591d86
Exists in master and in 1 other branch v2

added support for metadata update

Showing 2 changed files with 214 additions and 84 deletions   Show diff stats
... ... @@ -8,7 +8,7 @@ import (
8 8 const (
9 9 templateHttpErr500_EN = "An internal server error has occurred."
10 10 templateHttpErr500_RS = "Došlo je do greške na serveru."
11   - templateHttpErr400_EN = "Bad request: invalid request body."
  11 + templateHttpErr400_EN = "Bad request."
12 12 templateHttpErr400_RS = "Neispravan zahtev."
13 13 templateHttpErr404_EN = "Resource not found."
14 14 templateHttpErr404_RS = "Objekat nije pronadjen."
... ...
... ... @@ -6,13 +6,22 @@ import (
6 6 "fmt"
7 7 "io"
8 8 "net/http"
  9 + "os"
9 10 "sync"
  11 + "time"
10 12  
11 13 "gopkg.in/rana/ora.v4"
12 14 )
13 15  
14   -var mu = &sync.Mutex{}
15   -var payloads []Payload
  16 +var (
  17 + mu = &sync.Mutex{}
  18 + metadata map[string]Payload
  19 +
  20 + metadataDB *ora.Ses
  21 + activeProject string
  22 +
  23 + inited bool
  24 +)
16 25  
17 26 type LangMap map[string]map[string]string
18 27  
... ... @@ -35,7 +44,6 @@ type Translation struct {
35 44 }
36 45  
37 46 type Payload struct {
38   - Type string `json:"type"`
39 47 Method string `json:"method"`
40 48 Params map[string]string `json:"params"`
41 49 Lang []Translation `json:"lang"`
... ... @@ -47,21 +55,100 @@ type Payload struct {
47 55 Data interface{} `json:"data"`
48 56 }
49 57  
50   -// InitPayloadsMetaData loads all payloads in the payloads variable.
51   -// Returns an error if it fails.
52   -func InitPayloadsMetaData(db *ora.Ses, project string) error {
53   - payloads = make([]Payload, 0)
  58 +// LoadPayloadsdetaData loads all payloads' information into 'metadata' variable.
  59 +func LoadPayloadsMetadata(db *ora.Ses, project string, hotloading bool, hlPeriod int) error {
  60 + metadataDB = db
  61 + activeProject = project
54 62  
55   - jsonbuf, err := fetchJSON(db, project)
  63 + mu.Lock()
  64 + defer mu.Unlock()
  65 + err := initMetadata(project)
56 66 if err != nil {
57 67 return err
58 68 }
  69 + if hotloading {
  70 + go hotload(hlPeriod)
  71 + }
  72 + inited = true
  73 +
  74 + return nil
  75 +}
  76 +
  77 +func UpdateMetadataModels(md map[string][]byte) (upd, add int, err error) {
  78 + if !inited {
  79 + return 0, 0, errors.New("webutil: metadata not initialized but update was tried.")
  80 + }
  81 +
  82 + forUpdate := make([]string, 0)
  83 + forCreate := make([]string, 0)
  84 +
  85 + for k, _ := range md {
  86 + if _, exists := metadata[k]; exists {
  87 + forUpdate = append(forUpdate, k)
  88 + } else {
  89 + forCreate = append(forCreate, k)
  90 + }
  91 + }
  92 +
  93 + for _, k := range forUpdate {
  94 + fmt.Printf("for update: %s\n", k)
  95 + _, err := metadataDB.PrepAndExe(`update entities set
  96 + entity_model = :1
  97 + where entity_type = :2`,
  98 + string(md[k]),
  99 + k)
  100 +
  101 + if err != nil {
  102 + fmt.Printf("webutility: update metadata: prep and exe: %v\n", err)
  103 + continue
  104 + }
  105 + upd++
  106 + }
  107 +
  108 + for _, k := range forCreate {
  109 + fmt.Printf("for add: %s\n", k)
  110 + /*
  111 + _, err := metadataDB.PrepAndExe(`insert into entities
  112 + (projekat, metadata, entity_type, entity_model)
  113 + values(:1, :2, :3, :4)`,
  114 + activeProject, "", k, string(md[k]))
  115 +
  116 + if err != nil {
  117 + fmt.Printf("webutility: add metadata: prep and exe: %v\n", err)
  118 + continue
  119 + }
  120 + */
  121 + add++
  122 + }
  123 +
  124 + return upd, add, nil
  125 +}
  126 +
  127 +func GetMetadataForAllEntities() map[string]Payload {
  128 + return metadata
  129 +}
  130 +
  131 +func GetMetadataForEntityType(t string) Payload {
  132 + return metadata[t]
  133 +}
59 134  
  135 +func UpdateMetadata(entityType string, p *Payload) error {
  136 + md, err := json.Marshal(p)
  137 + if err != nil {
  138 + return err
  139 + }
  140 + fmt.Printf("md: %s %s\n", entityType, string(md))
60 141 mu.Lock()
61 142 defer mu.Unlock()
62   - json.Unmarshal(jsonbuf, &payloads)
63   - if len(payloads) == 0 {
64   - return errors.New("tables config is corrupt")
  143 + _, err = metadataDB.PrepAndExe(`update entities set
  144 + metadata = :1
  145 + where projekat = :2
  146 + and entity_type = :3`,
  147 + string(md),
  148 + activeProject,
  149 + entityType)
  150 + if err != nil {
  151 + return err
65 152 }
66 153 return nil
67 154 }
... ... @@ -72,100 +159,143 @@ func DecodeJSON(r io.Reader, v interface{}) error {
72 159 return json.NewDecoder(r).Decode(v)
73 160 }
74 161  
75   -// NewPayload returs a payload sceleton for provided table.
76   -func NewPayload(r *http.Request, table string) Payload {
77   - var pload Payload
78   -
  162 +// NewPayload returs a payload sceleton for entity described with etype.
  163 +func NewPayload(r *http.Request, etype string) Payload {
  164 + pload := metadata[etype]
79 165 pload.Method = r.Method + " " + r.RequestURI
80   - pload.Type = table
81   - if table != "" {
82   - pload.Params = make(map[string]string, 0)
83   - pload.Lang = translations(table)
84   - pload.Fields = fields(table)
85   - pload.IdField = id(table)
86   - pload.Correlations = correlations(table)
87   - }
88 166 return pload
89 167 }
90 168  
91   -// translations returns a slice of translations for a payload/table of ptype type.
92   -func translations(ptype string) []Translation {
93   - var translations []Translation
94   -
95   - for _, pload := range payloads {
96   - if pload.Type == ptype {
97   - for _, t := range pload.Lang {
98   - translations = append(translations, Translation{
99   - Language: t.Language,
100   - FieldsLabels: t.FieldsLabels,
101   - })
102   - }
103   - }
  169 +func initMetadata(project string) error {
  170 + metadataDB.SetCfg(metadataDB.Cfg().SetClob(ora.S))
  171 + stmt, err := metadataDB.Prep(`select
  172 + entity_type,
  173 + metadata
  174 + from entities
  175 + where projekat = `+fmt.Sprintf("'%s'", project),
  176 + ora.S,
  177 + ora.S)
  178 +
  179 + defer stmt.Close()
  180 + if err != nil {
  181 + return err
104 182 }
105 183  
106   - return translations
107   -}
  184 + rset, err := stmt.Qry()
  185 + if err != nil {
  186 + return err
  187 + }
108 188  
109   -// fields returns a slice of fields for a payload/table of ptype type.
110   -func fields(ptype string) []Field {
111   - var fields []Field
  189 + count := 0
  190 + success := 0
  191 + metadata = make(map[string]Payload)
  192 + for rset.Next() {
  193 + name := rset.Row[0].(string)
  194 + load := []byte(rset.Row[1].(string))
112 195  
113   - for _, pload := range payloads {
114   - if pload.Type == ptype {
115   - for _, f := range pload.Fields {
116   - fields = append(fields, f)
117   - }
  196 + p := Payload{}
  197 + err := json.Unmarshal(load, &p)
  198 + if err != nil {
  199 + fmt.Printf("couldn't init: '%s' metadata\n", name)
  200 + } else {
  201 + success++
  202 + metadata[name] = p
118 203 }
  204 + count++
119 205 }
  206 + fmt.Printf("webutility: successfully loaded %d/%d (%.1f%%) entities\n",
  207 + success, count, float32(success)/float32(count)*100.0)
120 208  
121   - return fields
  209 + return nil
122 210 }
123 211  
124   -// id returns the name of ID field of a payload/table of ptype type.
125   -func id(ptype string) string {
126   - for _, pload := range payloads {
127   - if pload.Type == ptype {
128   - return pload.IdField
  212 +func hotload(n int) {
  213 + entityScan := make(map[string]int64)
  214 + firstCheck := true
  215 + for {
  216 + time.Sleep(time.Duration(n) * time.Second)
  217 + stmt, err := metadataDB.Prep(`select
  218 + ora_rowscn,
  219 + entity_type
  220 + from entities where projekat = `+fmt.Sprintf("'%s'", activeProject),
  221 + ora.I64,
  222 + ora.S)
  223 + if err != nil {
  224 + fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err)
  225 + time.Sleep(time.Duration(n) * time.Second)
  226 + continue
129 227 }
130   - }
131   - return ""
132   -}
133 228  
134   -// correlations returns a slice of correlation fields for a payload/table of ptype type.
135   -func correlations(ptype string) []CorrelationField {
136   - var corr []CorrelationField
  229 + rset, err := stmt.Qry()
  230 + if err != nil {
  231 + fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err)
  232 + time.Sleep(time.Duration(n) * time.Second)
  233 + continue
  234 + }
137 235  
138   - for _, pload := range payloads {
139   - if pload.Type == ptype {
140   - for _, c := range pload.Correlations {
141   - corr = append(corr, c)
  236 + var toRefresh []string
  237 + for rset.Next() {
  238 + scanID := rset.Row[0].(int64)
  239 + entity := rset.Row[1].(string)
  240 + oldID, ok := entityScan[entity]
  241 + if !ok || oldID != scanID {
  242 + entityScan[entity] = scanID
  243 + toRefresh = append(toRefresh, entity)
142 244 }
143 245 }
144   - }
  246 + stmt.Close()
145 247  
146   - return corr
147   -}
  248 + if rset.Err() != nil {
  249 + fmt.Fprintf(os.Stderr, "hotload rset error: %v\n", rset.Err())
  250 + time.Sleep(time.Duration(n) * time.Second)
  251 + continue
  252 + }
148 253  
149   -// fetchJSON returns a byte slice of JSON configuration file from TABLES_CONFIG table.
150   -// Returns an error if it fails.
151   -func fetchJSON(db *ora.Ses, project string) ([]byte, error) {
152   - db.SetCfg(db.Cfg().SetClob(ora.S))
153   - stmt, err := db.Prep(`SELECT JSON_NCLOB FROM TABLES_CONFIG WHERE PROJEKAT = `+fmt.Sprintf("'%s'", project), ora.S)
154   - defer stmt.Close()
155   - if err != nil {
156   - return nil, err
  254 + if len(toRefresh) > 0 && !firstCheck {
  255 + mu.Lock()
  256 + refreshMetadata(toRefresh)
  257 + mu.Unlock()
  258 + }
  259 + if firstCheck {
  260 + firstCheck = false
  261 + }
157 262 }
  263 +}
158 264  
159   - rset, err := stmt.Qry()
160   - if err != nil {
161   - return nil, err
162   - }
  265 +func refreshMetadata(entities []string) {
  266 + for _, e := range entities {
  267 + //fmt.Printf("refreshing %s\n", e)
  268 + stmt, err := metadataDB.Prep(`select
  269 + metadata
  270 + from entities
  271 + where projekat = `+fmt.Sprintf("'%s'", activeProject)+
  272 + ` and entity_type = `+fmt.Sprintf("'%s'", e),
  273 + ora.S)
163 274  
164   - var data string
165   - if rset.Next() {
166   - data = rset.Row[0].(string)
167   - }
  275 + if err != nil {
  276 + fmt.Printf("webutility: refresh: prep: %v\n", err)
  277 + stmt.Close()
  278 + continue
  279 + }
168 280  
169   - //fmt.Println(data)
170   - return []byte(data), nil
  281 + rset, err := stmt.Qry()
  282 + if err != nil {
  283 + fmt.Printf("webutility: refresh: query: %v\n", err)
  284 + stmt.Close()
  285 + continue
  286 + }
  287 +
  288 + for rset.Next() {
  289 + load := []byte(rset.Row[0].(string))
  290 + p := Payload{}
  291 + err := json.Unmarshal(load, &p)
  292 + if err != nil {
  293 + fmt.Printf("couldn't refresh: '%s' metadata\n", e)
  294 + } else {
  295 + metadata[e] = p
  296 + //fmt.Printf("unmarshaled %s %v\n", e, metadata[e])
  297 + }
  298 + }
  299 + stmt.Close()
  300 + }
171 301 }
... ...