Marko Tikvić / webutility

Download as
Browse Code »

Commit 04651a369be120b6a66e831b115aef6c99d0ea1f

Authored by Marko Tikvić
1 parent 0e41f4d80b
Exists in master and in 1 other branch v2

new log name

Showing 1 changed file with 1 additions and 1 deletions   Show diff stats
1 package webutility 1 package webutility
2 2
3 import ( 3 import (
4 "encoding/json" 4 "encoding/json"
5 "errors" 5 "errors"
6 "fmt" 6 "fmt"
7 "io" 7 "io"
8 "net/http" 8 "net/http"
9 "os" 9 "os"
10 "sync" 10 "sync"
11 "time" 11 "time"
12 12
13 "git.to-net.rs/marko.tikvic/gologger" 13 "git.to-net.rs/marko.tikvic/gologger"
14 14
15 "gopkg.in/rana/ora.v4" 15 "gopkg.in/rana/ora.v4"
16 ) 16 )
17 17
18 var ( 18 var (
19 mu = &sync.Mutex{} 19 mu = &sync.Mutex{}
20 metadata = make(map[string]Payload) 20 metadata = make(map[string]Payload)
21 updateQue = make(map[string][]byte) 21 updateQue = make(map[string][]byte)
22 22
23 metadataDB *ora.Ses 23 metadataDB *ora.Ses
24 activeProject string 24 activeProject string
25 25
26 inited bool 26 inited bool
27 ) 27 )
28 28
29 var logger *gologger.Logger 29 var logger *gologger.Logger
30 30
31 func init() { 31 func init() {
32 var err error 32 var err error
33 logger, err = gologger.New("webutility", gologger.MaxLogSize100KB) 33 logger, err = gologger.New("metadata", gologger.MaxLogSize100KB)
34 if err != nil { 34 if err != nil {
35 fmt.Printf("webutility: %s\n", err.Error()) 35 fmt.Printf("webutility: %s\n", err.Error())
36 } 36 }
37 } 37 }
38 38
39 type LangMap map[string]map[string]string 39 type LangMap map[string]map[string]string
40 40
41 type Field struct { 41 type Field struct {
42 Parameter string `json:"param"` 42 Parameter string `json:"param"`
43 Type string `json:"type"` 43 Type string `json:"type"`
44 Visible bool `json:"visible"` 44 Visible bool `json:"visible"`
45 Editable bool `json:"editable"` 45 Editable bool `json:"editable"`
46 } 46 }
47 47
48 type CorrelationField struct { 48 type CorrelationField struct {
49 Result string `json:"result"` 49 Result string `json:"result"`
50 Elements []string `json:"elements"` 50 Elements []string `json:"elements"`
51 Type string `json:"type"` 51 Type string `json:"type"`
52 } 52 }
53 53
54 type Translation struct { 54 type Translation struct {
55 Language string `json:"language"` 55 Language string `json:"language"`
56 FieldsLabels map[string]string `json:"fieldsLabels"` 56 FieldsLabels map[string]string `json:"fieldsLabels"`
57 } 57 }
58 58
59 type Payload struct { 59 type Payload struct {
60 Method string `json:"method"` 60 Method string `json:"method"`
61 Params map[string]string `json:"params"` 61 Params map[string]string `json:"params"`
62 Lang []Translation `json:"lang"` 62 Lang []Translation `json:"lang"`
63 Fields []Field `json:"fields"` 63 Fields []Field `json:"fields"`
64 Correlations []CorrelationField `json:"correlationFields"` 64 Correlations []CorrelationField `json:"correlationFields"`
65 IdField string `json:"idField"` 65 IdField string `json:"idField"`
66 66
67 // Data holds JSON payload. It can't be used for itteration. 67 // Data holds JSON payload. It can't be used for itteration.
68 Data interface{} `json:"data"` 68 Data interface{} `json:"data"`
69 } 69 }
70 70
71 // LoadPayloadsdetaData loads all payloads' information into 'metadata' variable. 71 // LoadPayloadsdetaData loads all payloads' information into 'metadata' variable.
72 func LoadPayloadsMetadata(db *ora.Ses, project string, hotloading bool, hlPeriod int) error { 72 func LoadPayloadsMetadata(db *ora.Ses, project string, hotloading bool, hlPeriod int) error {
73 metadataDB = db 73 metadataDB = db
74 activeProject = project 74 activeProject = project
75 75
76 mu.Lock() 76 mu.Lock()
77 defer mu.Unlock() 77 defer mu.Unlock()
78 err := initMetadata(project) 78 err := initMetadata(project)
79 if err != nil { 79 if err != nil {
80 return err 80 return err
81 } 81 }
82 if hotloading { 82 if hotloading {
83 go hotload(hlPeriod) 83 go hotload(hlPeriod)
84 } 84 }
85 inited = true 85 inited = true
86 86
87 return nil 87 return nil
88 } 88 }
89 89
90 func GetMetadataForAllEntities() map[string]Payload { 90 func GetMetadataForAllEntities() map[string]Payload {
91 return metadata 91 return metadata
92 } 92 }
93 93
94 func GetMetadataForEntity(t string) (Payload, bool) { 94 func GetMetadataForEntity(t string) (Payload, bool) {
95 p, ok := metadata[t] 95 p, ok := metadata[t]
96 return p, ok 96 return p, ok
97 } 97 }
98 98
99 func QueEntityModelUpdate(entityType string, v interface{}) { 99 func QueEntityModelUpdate(entityType string, v interface{}) {
100 updateQue[entityType], _ = json.Marshal(v) 100 updateQue[entityType], _ = json.Marshal(v)
101 } 101 }
102 102
103 func UpdateEntityModels(command string) (total, upd, add int, err error) { 103 func UpdateEntityModels(command string) (total, upd, add int, err error) {
104 if command != "force" && command != "missing" { 104 if command != "force" && command != "missing" {
105 return total, 0, 0, errors.New("uknown command: " + command) 105 return total, 0, 0, errors.New("uknown command: " + command)
106 } 106 }
107 107
108 if !inited { 108 if !inited {
109 return 0, 0, 0, errors.New("webutil: metadata not initialized but update was tried.") 109 return 0, 0, 0, errors.New("webutil: metadata not initialized but update was tried.")
110 } 110 }
111 111
112 total = len(updateQue) 112 total = len(updateQue)
113 113
114 forUpdate := make([]string, 0) 114 forUpdate := make([]string, 0)
115 forAdd := make([]string, 0) 115 forAdd := make([]string, 0)
116 116
117 for k, _ := range updateQue { 117 for k, _ := range updateQue {
118 if _, exists := metadata[k]; exists { 118 if _, exists := metadata[k]; exists {
119 if command == "force" { 119 if command == "force" {
120 forUpdate = append(forUpdate, k) 120 forUpdate = append(forUpdate, k)
121 } 121 }
122 } else { 122 } else {
123 forAdd = append(forAdd, k) 123 forAdd = append(forAdd, k)
124 } 124 }
125 } 125 }
126 126
127 for _, k := range forUpdate { 127 for _, k := range forUpdate {
128 _, err := metadataDB.PrepAndExe(`update entities set 128 _, err := metadataDB.PrepAndExe(`update entities set
129 entity_model = :1 129 entity_model = :1
130 where entity_type = :2`, 130 where entity_type = :2`,
131 string(updateQue[k]), 131 string(updateQue[k]),
132 k) 132 k)
133 133
134 if err != nil { 134 if err != nil {
135 logger.Log("webutility: update metadata: prep and exe: %v\n", err) 135 logger.Log("webutility: update metadata: prep and exe: %v\n", err)
136 continue 136 continue
137 } 137 }
138 upd++ 138 upd++
139 logger.Log("webutility: updated %s payload model\n", k) 139 logger.Log("webutility: updated %s payload model\n", k)
140 } 140 }
141 141
142 blankPayload, _ := json.Marshal(Payload{}) 142 blankPayload, _ := json.Marshal(Payload{})
143 for _, k := range forAdd { 143 for _, k := range forAdd {
144 _, err := metadataDB.PrepAndExe(`insert into entities 144 _, err := metadataDB.PrepAndExe(`insert into entities
145 (projekat, metadata, entity_type, entity_model) 145 (projekat, metadata, entity_type, entity_model)
146 values(:1, :2, :3, :4)`, 146 values(:1, :2, :3, :4)`,
147 activeProject, 147 activeProject,
148 string(blankPayload), 148 string(blankPayload),
149 k, 149 k,
150 string(updateQue[k])) 150 string(updateQue[k]))
151 151
152 if err != nil { 152 if err != nil {
153 logger.Log("webutility: add metadata: prep and exe: %v\n", err) 153 logger.Log("webutility: add metadata: prep and exe: %v\n", err)
154 continue 154 continue
155 } 155 }
156 metadata[k] = Payload{} 156 metadata[k] = Payload{}
157 add++ 157 add++
158 logger.Log("webutility: added %s to the payload models\n", k) 158 logger.Log("webutility: added %s to the payload models\n", k)
159 159
160 } 160 }
161 161
162 return total, upd, add, nil 162 return total, upd, add, nil
163 } 163 }
164 164
165 func ModifyMetadataForEntity(entityType string, p *Payload) error { 165 func ModifyMetadataForEntity(entityType string, p *Payload) error {
166 md, err := json.Marshal(*p) 166 md, err := json.Marshal(*p)
167 if err != nil { 167 if err != nil {
168 return err 168 return err
169 } 169 }
170 170
171 mu.Lock() 171 mu.Lock()
172 defer mu.Unlock() 172 defer mu.Unlock()
173 _, err = metadataDB.PrepAndExe(`update entities set 173 _, err = metadataDB.PrepAndExe(`update entities set
174 metadata = :1 174 metadata = :1
175 where projekat = :2 175 where projekat = :2
176 and entity_type = :3`, 176 and entity_type = :3`,
177 string(md), 177 string(md),
178 activeProject, 178 activeProject,
179 entityType) 179 entityType)
180 if err != nil { 180 if err != nil {
181 return err 181 return err
182 } 182 }
183 return nil 183 return nil
184 } 184 }
185 185
186 func DeleteEntityModel(entityType string) error { 186 func DeleteEntityModel(entityType string) error {
187 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType) 187 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType)
188 if err == nil { 188 if err == nil {
189 mu.Lock() 189 mu.Lock()
190 delete(metadata, entityType) 190 delete(metadata, entityType)
191 mu.Unlock() 191 mu.Unlock()
192 } 192 }
193 return err 193 return err
194 } 194 }
195 195
196 // NewPayload returs a payload sceleton for entity described with etype. 196 // NewPayload returs a payload sceleton for entity described with etype.
197 func NewPayload(r *http.Request, etype string) Payload { 197 func NewPayload(r *http.Request, etype string) Payload {
198 pload := metadata[etype] 198 pload := metadata[etype]
199 pload.Method = r.Method + " " + r.RequestURI 199 pload.Method = r.Method + " " + r.RequestURI
200 return pload 200 return pload
201 } 201 }
202 202
203 // DecodeJSON decodes JSON data from r to v. 203 // DecodeJSON decodes JSON data from r to v.
204 // Returns an error if it fails. 204 // Returns an error if it fails.
205 func DecodeJSON(r io.Reader, v interface{}) error { 205 func DecodeJSON(r io.Reader, v interface{}) error {
206 return json.NewDecoder(r).Decode(v) 206 return json.NewDecoder(r).Decode(v)
207 } 207 }
208 208
209 func initMetadata(project string) error { 209 func initMetadata(project string) error {
210 metadataDB.SetCfg(metadataDB.Cfg().SetClob(ora.S)) 210 metadataDB.SetCfg(metadataDB.Cfg().SetClob(ora.S))
211 stmt, err := metadataDB.Prep(`select 211 stmt, err := metadataDB.Prep(`select
212 entity_type, 212 entity_type,
213 metadata 213 metadata
214 from entities 214 from entities
215 where projekat = `+fmt.Sprintf("'%s'", project), 215 where projekat = `+fmt.Sprintf("'%s'", project),
216 ora.S, 216 ora.S,
217 ora.S) 217 ora.S)
218 218
219 defer stmt.Close() 219 defer stmt.Close()
220 if err != nil { 220 if err != nil {
221 return err 221 return err
222 } 222 }
223 223
224 rset, err := stmt.Qry() 224 rset, err := stmt.Qry()
225 if err != nil { 225 if err != nil {
226 return err 226 return err
227 } 227 }
228 228
229 count := 0 229 count := 0
230 success := 0 230 success := 0
231 if len(metadata) > 0 { 231 if len(metadata) > 0 {
232 metadata = nil 232 metadata = nil
233 } 233 }
234 metadata = make(map[string]Payload) 234 metadata = make(map[string]Payload)
235 for rset.Next() { 235 for rset.Next() {
236 name := rset.Row[0].(string) 236 name := rset.Row[0].(string)
237 load := []byte(rset.Row[1].(string)) 237 load := []byte(rset.Row[1].(string))
238 238
239 p := Payload{} 239 p := Payload{}
240 err := json.Unmarshal(load, &p) 240 err := json.Unmarshal(load, &p)
241 if err != nil { 241 if err != nil {
242 logger.Log("couldn't init: '%s' metadata: %s\n%s\n", name, err.Error(), rset.Row[1].(string)) 242 logger.Log("couldn't init: '%s' metadata: %s\n%s\n", name, err.Error(), rset.Row[1].(string))
243 } else { 243 } else {
244 success++ 244 success++
245 metadata[name] = p 245 metadata[name] = p
246 } 246 }
247 count++ 247 count++
248 } 248 }
249 logger.Log("webutility: successfully loaded %d/%d (%.1f%%) entities\n", success, count, float32(success)/float32(count)*100.0) 249 logger.Log("webutility: successfully loaded %d/%d (%.1f%%) entities\n", success, count, float32(success)/float32(count)*100.0)
250 250
251 return nil 251 return nil
252 } 252 }
253 253
254 func hotload(n int) { 254 func hotload(n int) {
255 entityScan := make(map[string]int64) 255 entityScan := make(map[string]int64)
256 firstCheck := true 256 firstCheck := true
257 for { 257 for {
258 time.Sleep(time.Duration(n) * time.Second) 258 time.Sleep(time.Duration(n) * time.Second)
259 stmt, err := metadataDB.Prep(`select 259 stmt, err := metadataDB.Prep(`select
260 ora_rowscn, 260 ora_rowscn,
261 entity_type 261 entity_type
262 from entities where projekat = `+fmt.Sprintf("'%s'", activeProject), 262 from entities where projekat = `+fmt.Sprintf("'%s'", activeProject),
263 ora.I64, 263 ora.I64,
264 ora.S) 264 ora.S)
265 if err != nil { 265 if err != nil {
266 logger.Log("hotload failed: %v\n", err) 266 logger.Log("hotload failed: %v\n", err)
267 time.Sleep(time.Duration(n) * time.Second) 267 time.Sleep(time.Duration(n) * time.Second)
268 continue 268 continue
269 } 269 }
270 270
271 rset, err := stmt.Qry() 271 rset, err := stmt.Qry()
272 if err != nil { 272 if err != nil {
273 fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err) 273 fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err)
274 time.Sleep(time.Duration(n) * time.Second) 274 time.Sleep(time.Duration(n) * time.Second)
275 continue 275 continue
276 } 276 }
277 277
278 var toRefresh []string 278 var toRefresh []string
279 for rset.Next() { 279 for rset.Next() {
280 scanID := rset.Row[0].(int64) 280 scanID := rset.Row[0].(int64)
281 entity := rset.Row[1].(string) 281 entity := rset.Row[1].(string)
282 oldID, ok := entityScan[entity] 282 oldID, ok := entityScan[entity]
283 if !ok || oldID != scanID { 283 if !ok || oldID != scanID {
284 entityScan[entity] = scanID 284 entityScan[entity] = scanID
285 toRefresh = append(toRefresh, entity) 285 toRefresh = append(toRefresh, entity)
286 } 286 }
287 } 287 }
288 stmt.Close() 288 stmt.Close()
289 289
290 if rset.Err() != nil { 290 if rset.Err() != nil {
291 logger.Log("hotload rset error: %v\n", rset.Err()) 291 logger.Log("hotload rset error: %v\n", rset.Err())
292 time.Sleep(time.Duration(n) * time.Second) 292 time.Sleep(time.Duration(n) * time.Second)
293 continue 293 continue
294 } 294 }
295 295
296 if len(toRefresh) > 0 && !firstCheck { 296 if len(toRefresh) > 0 && !firstCheck {
297 mu.Lock() 297 mu.Lock()
298 refreshMetadata(toRefresh) 298 refreshMetadata(toRefresh)
299 mu.Unlock() 299 mu.Unlock()
300 } 300 }
301 if firstCheck { 301 if firstCheck {
302 firstCheck = false 302 firstCheck = false
303 } 303 }
304 } 304 }
305 } 305 }
306 306
307 func refreshMetadata(entities []string) { 307 func refreshMetadata(entities []string) {
308 for _, e := range entities { 308 for _, e := range entities {
309 fmt.Printf("refreshing %s\n", e) 309 fmt.Printf("refreshing %s\n", e)
310 stmt, err := metadataDB.Prep(`select 310 stmt, err := metadataDB.Prep(`select
311 metadata 311 metadata
312 from entities 312 from entities
313 where projekat = `+fmt.Sprintf("'%s'", activeProject)+ 313 where projekat = `+fmt.Sprintf("'%s'", activeProject)+
314 ` and entity_type = `+fmt.Sprintf("'%s'", e), 314 ` and entity_type = `+fmt.Sprintf("'%s'", e),
315 ora.S) 315 ora.S)
316 316
317 if err != nil { 317 if err != nil {
318 logger.Log("webutility: refresh: prep: %v\n", err) 318 logger.Log("webutility: refresh: prep: %v\n", err)
319 stmt.Close() 319 stmt.Close()
320 continue 320 continue
321 } 321 }
322 322
323 rset, err := stmt.Qry() 323 rset, err := stmt.Qry()
324 if err != nil { 324 if err != nil {
325 fmt.Printf("webutility: refresh: query: %v\n", err) 325 fmt.Printf("webutility: refresh: query: %v\n", err)
326 stmt.Close() 326 stmt.Close()
327 continue 327 continue
328 } 328 }
329 329
330 for rset.Next() { 330 for rset.Next() {
331 load := []byte(rset.Row[0].(string)) 331 load := []byte(rset.Row[0].(string))
332 p := Payload{} 332 p := Payload{}
333 err := json.Unmarshal(load, &p) 333 err := json.Unmarshal(load, &p)
334 if err != nil { 334 if err != nil {
335 logger.Log("couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), rset.Row[0].(string)) 335 logger.Log("couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), rset.Row[0].(string))
336 } else { 336 } else {
337 metadata[e] = p 337 metadata[e] = p
338 } 338 }
339 } 339 }
340 stmt.Close() 340 stmt.Close()
341 } 341 }
342 } 342 }
343 343