Marko Tikvić / webutility

Download as
Browse Code »

Commit 0e41f4d80b46656f9b9466db825c978785251989

Authored by Marko Tikvić
1 parent 0beb2d1fe7
Exists in master and in 1 other branch v2

refactored due to new gologger interface

Showing 1 changed file with 22 additions and 11 deletions   Show diff stats
1 package webutility 1 package webutility
2 2
3 import ( 3 import (
4 "encoding/json" 4 "encoding/json"
5 "errors" 5 "errors"
6 "fmt" 6 "fmt"
7 "io" 7 "io"
8 "net/http" 8 "net/http"
9 "os" 9 "os"
10 "sync" 10 "sync"
11 "time" 11 "time"
12 12
13 "git.to-net.rs/marko.tikvic/gologger"
14
13 "gopkg.in/rana/ora.v4" 15 "gopkg.in/rana/ora.v4"
14 ) 16 )
15 17
16 var ( 18 var (
17 mu = &sync.Mutex{} 19 mu = &sync.Mutex{}
18 metadata = make(map[string]Payload) 20 metadata = make(map[string]Payload)
19 updateQue = make(map[string][]byte) 21 updateQue = make(map[string][]byte)
20 22
21 metadataDB *ora.Ses 23 metadataDB *ora.Ses
22 activeProject string 24 activeProject string
23 25
24 inited bool 26 inited bool
25 ) 27 )
26 28
29 var logger *gologger.Logger
30
31 func init() {
32 var err error
33 logger, err = gologger.New("webutility", gologger.MaxLogSize100KB)
34 if err != nil {
35 fmt.Printf("webutility: %s\n", err.Error())
36 }
37 }
38
27 type LangMap map[string]map[string]string 39 type LangMap map[string]map[string]string
28 40
29 type Field struct { 41 type Field struct {
30 Parameter string `json:"param"` 42 Parameter string `json:"param"`
31 Type string `json:"type"` 43 Type string `json:"type"`
32 Visible bool `json:"visible"` 44 Visible bool `json:"visible"`
33 Editable bool `json:"editable"` 45 Editable bool `json:"editable"`
34 } 46 }
35 47
36 type CorrelationField struct { 48 type CorrelationField struct {
37 Result string `json:"result"` 49 Result string `json:"result"`
38 Elements []string `json:"elements"` 50 Elements []string `json:"elements"`
39 Type string `json:"type"` 51 Type string `json:"type"`
40 } 52 }
41 53
42 type Translation struct { 54 type Translation struct {
43 Language string `json:"language"` 55 Language string `json:"language"`
44 FieldsLabels map[string]string `json:"fieldsLabels"` 56 FieldsLabels map[string]string `json:"fieldsLabels"`
45 } 57 }
46 58
47 type Payload struct { 59 type Payload struct {
48 Method string `json:"method"` 60 Method string `json:"method"`
49 Params map[string]string `json:"params"` 61 Params map[string]string `json:"params"`
50 Lang []Translation `json:"lang"` 62 Lang []Translation `json:"lang"`
51 Fields []Field `json:"fields"` 63 Fields []Field `json:"fields"`
52 Correlations []CorrelationField `json:"correlationFields"` 64 Correlations []CorrelationField `json:"correlationFields"`
53 IdField string `json:"idField"` 65 IdField string `json:"idField"`
54 66
55 // Data holds JSON payload. It can't be used for itteration. 67 // Data holds JSON payload. It can't be used for itteration.
56 Data interface{} `json:"data"` 68 Data interface{} `json:"data"`
57 } 69 }
58 70
59 // LoadPayloadsdetaData loads all payloads' information into 'metadata' variable. 71 // LoadPayloadsdetaData loads all payloads' information into 'metadata' variable.
60 func LoadPayloadsMetadata(db *ora.Ses, project string, hotloading bool, hlPeriod int) error { 72 func LoadPayloadsMetadata(db *ora.Ses, project string, hotloading bool, hlPeriod int) error {
61 metadataDB = db 73 metadataDB = db
62 activeProject = project 74 activeProject = project
63 75
64 mu.Lock() 76 mu.Lock()
65 defer mu.Unlock() 77 defer mu.Unlock()
66 err := initMetadata(project) 78 err := initMetadata(project)
67 if err != nil { 79 if err != nil {
68 return err 80 return err
69 } 81 }
70 if hotloading { 82 if hotloading {
71 go hotload(hlPeriod) 83 go hotload(hlPeriod)
72 } 84 }
73 inited = true 85 inited = true
74 86
75 return nil 87 return nil
76 } 88 }
77 89
78 func GetMetadataForAllEntities() map[string]Payload { 90 func GetMetadataForAllEntities() map[string]Payload {
79 return metadata 91 return metadata
80 } 92 }
81 93
82 func GetMetadataForEntity(t string) (Payload, bool) { 94 func GetMetadataForEntity(t string) (Payload, bool) {
83 p, ok := metadata[t] 95 p, ok := metadata[t]
84 return p, ok 96 return p, ok
85 } 97 }
86 98
87 func QueEntityModelUpdate(entityType string, v interface{}) { 99 func QueEntityModelUpdate(entityType string, v interface{}) {
88 updateQue[entityType], _ = json.Marshal(v) 100 updateQue[entityType], _ = json.Marshal(v)
89 } 101 }
90 102
91 func UpdateEntityModels(command string) (total, upd, add int, err error) { 103 func UpdateEntityModels(command string) (total, upd, add int, err error) {
92 if command != "force" && command != "missing" { 104 if command != "force" && command != "missing" {
93 return total, 0, 0, errors.New("uknown command: " + command) 105 return total, 0, 0, errors.New("uknown command: " + command)
94 } 106 }
95 107
96 if !inited { 108 if !inited {
97 return 0, 0, 0, errors.New("webutil: metadata not initialized but update was tried.") 109 return 0, 0, 0, errors.New("webutil: metadata not initialized but update was tried.")
98 } 110 }
99 111
100 total = len(updateQue) 112 total = len(updateQue)
101 113
102 forUpdate := make([]string, 0) 114 forUpdate := make([]string, 0)
103 forAdd := make([]string, 0) 115 forAdd := make([]string, 0)
104 116
105 for k, _ := range updateQue { 117 for k, _ := range updateQue {
106 if _, exists := metadata[k]; exists { 118 if _, exists := metadata[k]; exists {
107 if command == "force" { 119 if command == "force" {
108 forUpdate = append(forUpdate, k) 120 forUpdate = append(forUpdate, k)
109 } 121 }
110 } else { 122 } else {
111 forAdd = append(forAdd, k) 123 forAdd = append(forAdd, k)
112 } 124 }
113 } 125 }
114 126
115 for _, k := range forUpdate { 127 for _, k := range forUpdate {
116 _, err := metadataDB.PrepAndExe(`update entities set 128 _, err := metadataDB.PrepAndExe(`update entities set
117 entity_model = :1 129 entity_model = :1
118 where entity_type = :2`, 130 where entity_type = :2`,
119 string(updateQue[k]), 131 string(updateQue[k]),
120 k) 132 k)
121 133
122 if err != nil { 134 if err != nil {
123 fmt.Printf("webutility: update metadata: prep and exe: %v\n", err) 135 logger.Log("webutility: update metadata: prep and exe: %v\n", err)
124 continue 136 continue
125 } 137 }
126 upd++ 138 upd++
127 fmt.Printf("webutility: updated %s payload model\n", k) 139 logger.Log("webutility: updated %s payload model\n", k)
128 } 140 }
129 141
130 blankPayload, _ := json.Marshal(Payload{}) 142 blankPayload, _ := json.Marshal(Payload{})
131 for _, k := range forAdd { 143 for _, k := range forAdd {
132 _, err := metadataDB.PrepAndExe(`insert into entities 144 _, err := metadataDB.PrepAndExe(`insert into entities
133 (projekat, metadata, entity_type, entity_model) 145 (projekat, metadata, entity_type, entity_model)
134 values(:1, :2, :3, :4)`, 146 values(:1, :2, :3, :4)`,
135 activeProject, 147 activeProject,
136 string(blankPayload), 148 string(blankPayload),
137 k, 149 k,
138 string(updateQue[k])) 150 string(updateQue[k]))
139 151
140 if err != nil { 152 if err != nil {
141 fmt.Printf("webutility: add metadata: prep and exe: %v\n", err) 153 logger.Log("webutility: add metadata: prep and exe: %v\n", err)
142 continue 154 continue
143 } 155 }
144 metadata[k] = Payload{} 156 metadata[k] = Payload{}
145 add++ 157 add++
146 fmt.Printf("webutility: added %s to the payload models\n", k) 158 logger.Log("webutility: added %s to the payload models\n", k)
147 159
148 } 160 }
149 161
150 return total, upd, add, nil 162 return total, upd, add, nil
151 } 163 }
152 164
153 func ModifyMetadataForEntity(entityType string, p *Payload) error { 165 func ModifyMetadataForEntity(entityType string, p *Payload) error {
154 md, err := json.Marshal(*p) 166 md, err := json.Marshal(*p)
155 if err != nil { 167 if err != nil {
156 return err 168 return err
157 } 169 }
158 170
159 mu.Lock() 171 mu.Lock()
160 defer mu.Unlock() 172 defer mu.Unlock()
161 _, err = metadataDB.PrepAndExe(`update entities set 173 _, err = metadataDB.PrepAndExe(`update entities set
162 metadata = :1 174 metadata = :1
163 where projekat = :2 175 where projekat = :2
164 and entity_type = :3`, 176 and entity_type = :3`,
165 string(md), 177 string(md),
166 activeProject, 178 activeProject,
167 entityType) 179 entityType)
168 if err != nil { 180 if err != nil {
169 return err 181 return err
170 } 182 }
171 return nil 183 return nil
172 } 184 }
173 185
174 func DeleteEntityModel(entityType string) error { 186 func DeleteEntityModel(entityType string) error {
175 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType) 187 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType)
176 if err == nil { 188 if err == nil {
177 mu.Lock() 189 mu.Lock()
178 delete(metadata, entityType) 190 delete(metadata, entityType)
179 mu.Unlock() 191 mu.Unlock()
180 } 192 }
181 return err 193 return err
182 } 194 }
183 195
184 // NewPayload returs a payload sceleton for entity described with etype. 196 // NewPayload returs a payload sceleton for entity described with etype.
185 func NewPayload(r *http.Request, etype string) Payload { 197 func NewPayload(r *http.Request, etype string) Payload {
186 pload := metadata[etype] 198 pload := metadata[etype]
187 pload.Method = r.Method + " " + r.RequestURI 199 pload.Method = r.Method + " " + r.RequestURI
188 return pload 200 return pload
189 } 201 }
190 202
191 // DecodeJSON decodes JSON data from r to v. 203 // DecodeJSON decodes JSON data from r to v.
192 // Returns an error if it fails. 204 // Returns an error if it fails.
193 func DecodeJSON(r io.Reader, v interface{}) error { 205 func DecodeJSON(r io.Reader, v interface{}) error {
194 return json.NewDecoder(r).Decode(v) 206 return json.NewDecoder(r).Decode(v)
195 } 207 }
196 208
197 func initMetadata(project string) error { 209 func initMetadata(project string) error {
198 metadataDB.SetCfg(metadataDB.Cfg().SetClob(ora.S)) 210 metadataDB.SetCfg(metadataDB.Cfg().SetClob(ora.S))
199 stmt, err := metadataDB.Prep(`select 211 stmt, err := metadataDB.Prep(`select
200 entity_type, 212 entity_type,
201 metadata 213 metadata
202 from entities 214 from entities
203 where projekat = `+fmt.Sprintf("'%s'", project), 215 where projekat = `+fmt.Sprintf("'%s'", project),
204 ora.S, 216 ora.S,
205 ora.S) 217 ora.S)
206 218
207 defer stmt.Close() 219 defer stmt.Close()
208 if err != nil { 220 if err != nil {
209 return err 221 return err
210 } 222 }
211 223
212 rset, err := stmt.Qry() 224 rset, err := stmt.Qry()
213 if err != nil { 225 if err != nil {
214 return err 226 return err
215 } 227 }
216 228
217 count := 0 229 count := 0
218 success := 0 230 success := 0
219 if len(metadata) > 0 { 231 if len(metadata) > 0 {
220 metadata = nil 232 metadata = nil
221 } 233 }
222 metadata = make(map[string]Payload) 234 metadata = make(map[string]Payload)
223 for rset.Next() { 235 for rset.Next() {
224 name := rset.Row[0].(string) 236 name := rset.Row[0].(string)
225 load := []byte(rset.Row[1].(string)) 237 load := []byte(rset.Row[1].(string))
226 238
227 p := Payload{} 239 p := Payload{}
228 err := json.Unmarshal(load, &p) 240 err := json.Unmarshal(load, &p)
229 if err != nil { 241 if err != nil {
230 fmt.Printf("couldn't init: '%s' metadata\n", name) 242 logger.Log("couldn't init: '%s' metadata: %s\n%s\n", name, err.Error(), rset.Row[1].(string))
231 } else { 243 } else {
232 success++ 244 success++
233 metadata[name] = p 245 metadata[name] = p
234 } 246 }
235 count++ 247 count++
236 } 248 }
237 fmt.Printf("webutility: successfully loaded %d/%d (%.1f%%) entities\n", 249 logger.Log("webutility: successfully loaded %d/%d (%.1f%%) entities\n", success, count, float32(success)/float32(count)*100.0)
238 success, count, float32(success)/float32(count)*100.0)
239 250
240 return nil 251 return nil
241 } 252 }
242 253
243 func hotload(n int) { 254 func hotload(n int) {
244 entityScan := make(map[string]int64) 255 entityScan := make(map[string]int64)
245 firstCheck := true 256 firstCheck := true
246 for { 257 for {
247 time.Sleep(time.Duration(n) * time.Second) 258 time.Sleep(time.Duration(n) * time.Second)
248 stmt, err := metadataDB.Prep(`select 259 stmt, err := metadataDB.Prep(`select
249 ora_rowscn, 260 ora_rowscn,
250 entity_type 261 entity_type
251 from entities where projekat = `+fmt.Sprintf("'%s'", activeProject), 262 from entities where projekat = `+fmt.Sprintf("'%s'", activeProject),
252 ora.I64, 263 ora.I64,
253 ora.S) 264 ora.S)
254 if err != nil { 265 if err != nil {
255 fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err) 266 logger.Log("hotload failed: %v\n", err)
256 time.Sleep(time.Duration(n) * time.Second) 267 time.Sleep(time.Duration(n) * time.Second)
257 continue 268 continue
258 } 269 }
259 270
260 rset, err := stmt.Qry() 271 rset, err := stmt.Qry()
261 if err != nil { 272 if err != nil {
262 fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err) 273 fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err)
263 time.Sleep(time.Duration(n) * time.Second) 274 time.Sleep(time.Duration(n) * time.Second)
264 continue 275 continue
265 } 276 }
266 277
267 var toRefresh []string 278 var toRefresh []string
268 for rset.Next() { 279 for rset.Next() {
269 scanID := rset.Row[0].(int64) 280 scanID := rset.Row[0].(int64)
270 entity := rset.Row[1].(string) 281 entity := rset.Row[1].(string)
271 oldID, ok := entityScan[entity] 282 oldID, ok := entityScan[entity]
272 if !ok || oldID != scanID { 283 if !ok || oldID != scanID {
273 entityScan[entity] = scanID 284 entityScan[entity] = scanID
274 toRefresh = append(toRefresh, entity) 285 toRefresh = append(toRefresh, entity)
275 } 286 }
276 } 287 }
277 stmt.Close() 288 stmt.Close()
278 289
279 if rset.Err() != nil { 290 if rset.Err() != nil {
280 fmt.Fprintf(os.Stderr, "hotload rset error: %v\n", rset.Err()) 291 logger.Log("hotload rset error: %v\n", rset.Err())
281 time.Sleep(time.Duration(n) * time.Second) 292 time.Sleep(time.Duration(n) * time.Second)
282 continue 293 continue
283 } 294 }
284 295
285 if len(toRefresh) > 0 && !firstCheck { 296 if len(toRefresh) > 0 && !firstCheck {
286 mu.Lock() 297 mu.Lock()
287 refreshMetadata(toRefresh) 298 refreshMetadata(toRefresh)
288 mu.Unlock() 299 mu.Unlock()
289 } 300 }
290 if firstCheck { 301 if firstCheck {
291 firstCheck = false 302 firstCheck = false
292 } 303 }
293 } 304 }
294 } 305 }
295 306
296 func refreshMetadata(entities []string) { 307 func refreshMetadata(entities []string) {
297 for _, e := range entities { 308 for _, e := range entities {
298 fmt.Printf("refreshing %s\n", e) 309 fmt.Printf("refreshing %s\n", e)
299 stmt, err := metadataDB.Prep(`select 310 stmt, err := metadataDB.Prep(`select
300 metadata 311 metadata
301 from entities 312 from entities
302 where projekat = `+fmt.Sprintf("'%s'", activeProject)+ 313 where projekat = `+fmt.Sprintf("'%s'", activeProject)+
303 ` and entity_type = `+fmt.Sprintf("'%s'", e), 314 ` and entity_type = `+fmt.Sprintf("'%s'", e),
304 ora.S) 315 ora.S)
305 316
306 if err != nil { 317 if err != nil {
307 fmt.Printf("webutility: refresh: prep: %v\n", err) 318 logger.Log("webutility: refresh: prep: %v\n", err)
308 stmt.Close() 319 stmt.Close()
309 continue 320 continue
310 } 321 }
311 322
312 rset, err := stmt.Qry() 323 rset, err := stmt.Qry()
313 if err != nil { 324 if err != nil {
314 fmt.Printf("webutility: refresh: query: %v\n", err) 325 fmt.Printf("webutility: refresh: query: %v\n", err)
315 stmt.Close() 326 stmt.Close()
316 continue 327 continue
317 } 328 }
318 329
319 for rset.Next() { 330 for rset.Next() {
320 load := []byte(rset.Row[0].(string)) 331 load := []byte(rset.Row[0].(string))
321 p := Payload{} 332 p := Payload{}
322 err := json.Unmarshal(load, &p) 333 err := json.Unmarshal(load, &p)
323 if err != nil { 334 if err != nil {
324 fmt.Printf("couldn't refresh: '%s' metadata\n", e) 335 logger.Log("couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), rset.Row[0].(string))
325 } else { 336 } else {
326 metadata[e] = p 337 metadata[e] = p
327 } 338 }
328 } 339 }
329 stmt.Close() 340 stmt.Close()
330 } 341 }
331 } 342 }