Marko Tikvić / webutility

Download as
Browse Code »

Commit 61efd58cdb2348b9133fd497bc5f0133bf4f3cb1

Authored by Marko Tikvić
1 parent 04651a369b
Exists in master and in 1 other branch v2

Put hotload enabling in separate function

Showing 1 changed file with 7 additions and 4 deletions   Show diff stats
1 package webutility 1 package webutility
2 2
3 import ( 3 import (
4 "encoding/json" 4 "encoding/json"
5 "errors" 5 "errors"
6 "fmt" 6 "fmt"
7 "io" 7 "io"
8 "net/http" 8 "net/http"
9 "os" 9 "os"
10 "sync" 10 "sync"
11 "time" 11 "time"
12 12
13 "git.to-net.rs/marko.tikvic/gologger" 13 "git.to-net.rs/marko.tikvic/gologger"
14 14
15 "gopkg.in/rana/ora.v4" 15 "gopkg.in/rana/ora.v4"
16 ) 16 )
17 17
18 var ( 18 var (
19 mu = &sync.Mutex{} 19 mu = &sync.Mutex{}
20 metadata = make(map[string]Payload) 20 metadata = make(map[string]Payload)
21 updateQue = make(map[string][]byte) 21 updateQue = make(map[string][]byte)
22 22
23 metadataDB *ora.Ses 23 metadataDB *ora.Ses
24 activeProject string 24 activeProject string
25 25
26 inited bool 26 inited bool
27 ) 27 )
28 28
29 var logger *gologger.Logger 29 var logger *gologger.Logger
30 30
31 func init() { 31 func init() {
32 var err error 32 var err error
33 logger, err = gologger.New("metadata", gologger.MaxLogSize100KB) 33 logger, err = gologger.New("metadata", gologger.MaxLogSize100KB)
34 if err != nil { 34 if err != nil {
35 fmt.Printf("webutility: %s\n", err.Error()) 35 fmt.Printf("webutility: %s\n", err.Error())
36 } 36 }
37 } 37 }
38 38
39 type LangMap map[string]map[string]string 39 type LangMap map[string]map[string]string
40 40
41 type Field struct { 41 type Field struct {
42 Parameter string `json:"param"` 42 Parameter string `json:"param"`
43 Type string `json:"type"` 43 Type string `json:"type"`
44 Visible bool `json:"visible"` 44 Visible bool `json:"visible"`
45 Editable bool `json:"editable"` 45 Editable bool `json:"editable"`
46 } 46 }
47 47
48 type CorrelationField struct { 48 type CorrelationField struct {
49 Result string `json:"result"` 49 Result string `json:"result"`
50 Elements []string `json:"elements"` 50 Elements []string `json:"elements"`
51 Type string `json:"type"` 51 Type string `json:"type"`
52 } 52 }
53 53
54 type Translation struct { 54 type Translation struct {
55 Language string `json:"language"` 55 Language string `json:"language"`
56 FieldsLabels map[string]string `json:"fieldsLabels"` 56 FieldsLabels map[string]string `json:"fieldsLabels"`
57 } 57 }
58 58
59 type Payload struct { 59 type Payload struct {
60 Method string `json:"method"` 60 Method string `json:"method"`
61 Params map[string]string `json:"params"` 61 Params map[string]string `json:"params"`
62 Lang []Translation `json:"lang"` 62 Lang []Translation `json:"lang"`
63 Fields []Field `json:"fields"` 63 Fields []Field `json:"fields"`
64 Correlations []CorrelationField `json:"correlationFields"` 64 Correlations []CorrelationField `json:"correlationFields"`
65 IdField string `json:"idField"` 65 IdField string `json:"idField"`
66 66
67 // Data holds JSON payload. It can't be used for itteration. 67 // Data holds JSON payload. It can't be used for itteration.
68 Data interface{} `json:"data"` 68 Data interface{} `json:"data"`
69 } 69 }
70 70
71 // LoadPayloadsdetaData loads all payloads' information into 'metadata' variable. 71 // LoadPayloadsdetaData loads all payloads' information into 'metadata' variable.
72 func LoadPayloadsMetadata(db *ora.Ses, project string, hotloading bool, hlPeriod int) error { 72 func LoadPayloadsMetadata(db *ora.Ses, project string) error {
73 metadataDB = db 73 metadataDB = db
74 activeProject = project 74 activeProject = project
75 75
76 mu.Lock() 76 mu.Lock()
77 defer mu.Unlock() 77 defer mu.Unlock()
78 err := initMetadata(project) 78 err := initMetadata(project)
79 if err != nil { 79 if err != nil {
80 return err 80 return err
81 } 81 }
82 if hotloading {
83 go hotload(hlPeriod)
84 }
85 inited = true 82 inited = true
86 83
87 return nil 84 return nil
88 } 85 }
89 86
87 func EnableHotloading(interval int) {
88 if interval > 0 {
89 go hotload(interval)
90 }
91 }
92
90 func GetMetadataForAllEntities() map[string]Payload { 93 func GetMetadataForAllEntities() map[string]Payload {
91 return metadata 94 return metadata
92 } 95 }
93 96
94 func GetMetadataForEntity(t string) (Payload, bool) { 97 func GetMetadataForEntity(t string) (Payload, bool) {
95 p, ok := metadata[t] 98 p, ok := metadata[t]
96 return p, ok 99 return p, ok
97 } 100 }
98 101
99 func QueEntityModelUpdate(entityType string, v interface{}) { 102 func QueEntityModelUpdate(entityType string, v interface{}) {
100 updateQue[entityType], _ = json.Marshal(v) 103 updateQue[entityType], _ = json.Marshal(v)
101 } 104 }
102 105
103 func UpdateEntityModels(command string) (total, upd, add int, err error) { 106 func UpdateEntityModels(command string) (total, upd, add int, err error) {
104 if command != "force" && command != "missing" { 107 if command != "force" && command != "missing" {
105 return total, 0, 0, errors.New("uknown command: " + command) 108 return total, 0, 0, errors.New("uknown command: " + command)
106 } 109 }
107 110
108 if !inited { 111 if !inited {
109 return 0, 0, 0, errors.New("webutil: metadata not initialized but update was tried.") 112 return 0, 0, 0, errors.New("webutil: metadata not initialized but update was tried.")
110 } 113 }
111 114
112 total = len(updateQue) 115 total = len(updateQue)
113 116
114 forUpdate := make([]string, 0) 117 forUpdate := make([]string, 0)
115 forAdd := make([]string, 0) 118 forAdd := make([]string, 0)
116 119
117 for k, _ := range updateQue { 120 for k, _ := range updateQue {
118 if _, exists := metadata[k]; exists { 121 if _, exists := metadata[k]; exists {
119 if command == "force" { 122 if command == "force" {
120 forUpdate = append(forUpdate, k) 123 forUpdate = append(forUpdate, k)
121 } 124 }
122 } else { 125 } else {
123 forAdd = append(forAdd, k) 126 forAdd = append(forAdd, k)
124 } 127 }
125 } 128 }
126 129
127 for _, k := range forUpdate { 130 for _, k := range forUpdate {
128 _, err := metadataDB.PrepAndExe(`update entities set 131 _, err := metadataDB.PrepAndExe(`update entities set
129 entity_model = :1 132 entity_model = :1
130 where entity_type = :2`, 133 where entity_type = :2`,
131 string(updateQue[k]), 134 string(updateQue[k]),
132 k) 135 k)
133 136
134 if err != nil { 137 if err != nil {
135 logger.Log("webutility: update metadata: prep and exe: %v\n", err) 138 logger.Log("webutility: update metadata: prep and exe: %v\n", err)
136 continue 139 continue
137 } 140 }
138 upd++ 141 upd++
139 logger.Log("webutility: updated %s payload model\n", k) 142 logger.Log("webutility: updated %s payload model\n", k)
140 } 143 }
141 144
142 blankPayload, _ := json.Marshal(Payload{}) 145 blankPayload, _ := json.Marshal(Payload{})
143 for _, k := range forAdd { 146 for _, k := range forAdd {
144 _, err := metadataDB.PrepAndExe(`insert into entities 147 _, err := metadataDB.PrepAndExe(`insert into entities
145 (projekat, metadata, entity_type, entity_model) 148 (projekat, metadata, entity_type, entity_model)
146 values(:1, :2, :3, :4)`, 149 values(:1, :2, :3, :4)`,
147 activeProject, 150 activeProject,
148 string(blankPayload), 151 string(blankPayload),
149 k, 152 k,
150 string(updateQue[k])) 153 string(updateQue[k]))
151 154
152 if err != nil { 155 if err != nil {
153 logger.Log("webutility: add metadata: prep and exe: %v\n", err) 156 logger.Log("webutility: add metadata: prep and exe: %v\n", err)
154 continue 157 continue
155 } 158 }
156 metadata[k] = Payload{} 159 metadata[k] = Payload{}
157 add++ 160 add++
158 logger.Log("webutility: added %s to the payload models\n", k) 161 logger.Log("webutility: added %s to the payload models\n", k)
159 162
160 } 163 }
161 164
162 return total, upd, add, nil 165 return total, upd, add, nil
163 } 166 }
164 167
165 func ModifyMetadataForEntity(entityType string, p *Payload) error { 168 func ModifyMetadataForEntity(entityType string, p *Payload) error {
166 md, err := json.Marshal(*p) 169 md, err := json.Marshal(*p)
167 if err != nil { 170 if err != nil {
168 return err 171 return err
169 } 172 }
170 173
171 mu.Lock() 174 mu.Lock()
172 defer mu.Unlock() 175 defer mu.Unlock()
173 _, err = metadataDB.PrepAndExe(`update entities set 176 _, err = metadataDB.PrepAndExe(`update entities set
174 metadata = :1 177 metadata = :1
175 where projekat = :2 178 where projekat = :2
176 and entity_type = :3`, 179 and entity_type = :3`,
177 string(md), 180 string(md),
178 activeProject, 181 activeProject,
179 entityType) 182 entityType)
180 if err != nil { 183 if err != nil {
181 return err 184 return err
182 } 185 }
183 return nil 186 return nil
184 } 187 }
185 188
186 func DeleteEntityModel(entityType string) error { 189 func DeleteEntityModel(entityType string) error {
187 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType) 190 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType)
188 if err == nil { 191 if err == nil {
189 mu.Lock() 192 mu.Lock()
190 delete(metadata, entityType) 193 delete(metadata, entityType)
191 mu.Unlock() 194 mu.Unlock()
192 } 195 }
193 return err 196 return err
194 } 197 }
195 198
196 // NewPayload returs a payload sceleton for entity described with etype. 199 // NewPayload returs a payload sceleton for entity described with etype.
197 func NewPayload(r *http.Request, etype string) Payload { 200 func NewPayload(r *http.Request, etype string) Payload {
198 pload := metadata[etype] 201 pload := metadata[etype]
199 pload.Method = r.Method + " " + r.RequestURI 202 pload.Method = r.Method + " " + r.RequestURI
200 return pload 203 return pload
201 } 204 }
202 205
203 // DecodeJSON decodes JSON data from r to v. 206 // DecodeJSON decodes JSON data from r to v.
204 // Returns an error if it fails. 207 // Returns an error if it fails.
205 func DecodeJSON(r io.Reader, v interface{}) error { 208 func DecodeJSON(r io.Reader, v interface{}) error {
206 return json.NewDecoder(r).Decode(v) 209 return json.NewDecoder(r).Decode(v)
207 } 210 }
208 211
209 func initMetadata(project string) error { 212 func initMetadata(project string) error {
210 metadataDB.SetCfg(metadataDB.Cfg().SetClob(ora.S)) 213 metadataDB.SetCfg(metadataDB.Cfg().SetClob(ora.S))
211 stmt, err := metadataDB.Prep(`select 214 stmt, err := metadataDB.Prep(`select
212 entity_type, 215 entity_type,
213 metadata 216 metadata
214 from entities 217 from entities
215 where projekat = `+fmt.Sprintf("'%s'", project), 218 where projekat = `+fmt.Sprintf("'%s'", project),
216 ora.S, 219 ora.S,
217 ora.S) 220 ora.S)
218 221
219 defer stmt.Close() 222 defer stmt.Close()
220 if err != nil { 223 if err != nil {
221 return err 224 return err
222 } 225 }
223 226
224 rset, err := stmt.Qry() 227 rset, err := stmt.Qry()
225 if err != nil { 228 if err != nil {
226 return err 229 return err
227 } 230 }
228 231
229 count := 0 232 count := 0
230 success := 0 233 success := 0
231 if len(metadata) > 0 { 234 if len(metadata) > 0 {
232 metadata = nil 235 metadata = nil
233 } 236 }
234 metadata = make(map[string]Payload) 237 metadata = make(map[string]Payload)
235 for rset.Next() { 238 for rset.Next() {
236 name := rset.Row[0].(string) 239 name := rset.Row[0].(string)
237 load := []byte(rset.Row[1].(string)) 240 load := []byte(rset.Row[1].(string))
238 241
239 p := Payload{} 242 p := Payload{}
240 err := json.Unmarshal(load, &p) 243 err := json.Unmarshal(load, &p)
241 if err != nil { 244 if err != nil {
242 logger.Log("couldn't init: '%s' metadata: %s\n%s\n", name, err.Error(), rset.Row[1].(string)) 245 logger.Log("couldn't init: '%s' metadata: %s\n%s\n", name, err.Error(), rset.Row[1].(string))
243 } else { 246 } else {
244 success++ 247 success++
245 metadata[name] = p 248 metadata[name] = p
246 } 249 }
247 count++ 250 count++
248 } 251 }
249 logger.Log("webutility: successfully loaded %d/%d (%.1f%%) entities\n", success, count, float32(success)/float32(count)*100.0) 252 logger.Log("webutility: successfully loaded %d/%d (%.1f%%) entities\n", success, count, float32(success)/float32(count)*100.0)
250 253
251 return nil 254 return nil
252 } 255 }
253 256
254 func hotload(n int) { 257 func hotload(n int) {
255 entityScan := make(map[string]int64) 258 entityScan := make(map[string]int64)
256 firstCheck := true 259 firstCheck := true
257 for { 260 for {
258 time.Sleep(time.Duration(n) * time.Second) 261 time.Sleep(time.Duration(n) * time.Second)
259 stmt, err := metadataDB.Prep(`select 262 stmt, err := metadataDB.Prep(`select
260 ora_rowscn, 263 ora_rowscn,
261 entity_type 264 entity_type
262 from entities where projekat = `+fmt.Sprintf("'%s'", activeProject), 265 from entities where projekat = `+fmt.Sprintf("'%s'", activeProject),
263 ora.I64, 266 ora.I64,
264 ora.S) 267 ora.S)
265 if err != nil { 268 if err != nil {
266 logger.Log("hotload failed: %v\n", err) 269 logger.Log("hotload failed: %v\n", err)
267 time.Sleep(time.Duration(n) * time.Second) 270 time.Sleep(time.Duration(n) * time.Second)
268 continue 271 continue
269 } 272 }
270 273
271 rset, err := stmt.Qry() 274 rset, err := stmt.Qry()
272 if err != nil { 275 if err != nil {
273 fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err) 276 fmt.Fprintf(os.Stderr, "hotload failed: %v\n", err)
274 time.Sleep(time.Duration(n) * time.Second) 277 time.Sleep(time.Duration(n) * time.Second)
275 continue 278 continue
276 } 279 }
277 280
278 var toRefresh []string 281 var toRefresh []string
279 for rset.Next() { 282 for rset.Next() {
280 scanID := rset.Row[0].(int64) 283 scanID := rset.Row[0].(int64)
281 entity := rset.Row[1].(string) 284 entity := rset.Row[1].(string)
282 oldID, ok := entityScan[entity] 285 oldID, ok := entityScan[entity]
283 if !ok || oldID != scanID { 286 if !ok || oldID != scanID {
284 entityScan[entity] = scanID 287 entityScan[entity] = scanID
285 toRefresh = append(toRefresh, entity) 288 toRefresh = append(toRefresh, entity)
286 } 289 }
287 } 290 }
288 stmt.Close() 291 stmt.Close()
289 292
290 if rset.Err() != nil { 293 if rset.Err() != nil {
291 logger.Log("hotload rset error: %v\n", rset.Err()) 294 logger.Log("hotload rset error: %v\n", rset.Err())
292 time.Sleep(time.Duration(n) * time.Second) 295 time.Sleep(time.Duration(n) * time.Second)
293 continue 296 continue
294 } 297 }
295 298
296 if len(toRefresh) > 0 && !firstCheck { 299 if len(toRefresh) > 0 && !firstCheck {
297 mu.Lock() 300 mu.Lock()
298 refreshMetadata(toRefresh) 301 refreshMetadata(toRefresh)
299 mu.Unlock() 302 mu.Unlock()
300 } 303 }
301 if firstCheck { 304 if firstCheck {
302 firstCheck = false 305 firstCheck = false
303 } 306 }
304 } 307 }
305 } 308 }
306 309
307 func refreshMetadata(entities []string) { 310 func refreshMetadata(entities []string) {
308 for _, e := range entities { 311 for _, e := range entities {
309 fmt.Printf("refreshing %s\n", e) 312 fmt.Printf("refreshing %s\n", e)
310 stmt, err := metadataDB.Prep(`select 313 stmt, err := metadataDB.Prep(`select
311 metadata 314 metadata
312 from entities 315 from entities
313 where projekat = `+fmt.Sprintf("'%s'", activeProject)+ 316 where projekat = `+fmt.Sprintf("'%s'", activeProject)+
314 ` and entity_type = `+fmt.Sprintf("'%s'", e), 317 ` and entity_type = `+fmt.Sprintf("'%s'", e),
315 ora.S) 318 ora.S)
316 319
317 if err != nil { 320 if err != nil {
318 logger.Log("webutility: refresh: prep: %v\n", err) 321 logger.Log("webutility: refresh: prep: %v\n", err)
319 stmt.Close() 322 stmt.Close()
320 continue 323 continue
321 } 324 }
322 325
323 rset, err := stmt.Qry() 326 rset, err := stmt.Qry()
324 if err != nil { 327 if err != nil {
325 fmt.Printf("webutility: refresh: query: %v\n", err) 328 fmt.Printf("webutility: refresh: query: %v\n", err)
326 stmt.Close() 329 stmt.Close()
327 continue 330 continue
328 } 331 }
329 332
330 for rset.Next() { 333 for rset.Next() {
331 load := []byte(rset.Row[0].(string)) 334 load := []byte(rset.Row[0].(string))
332 p := Payload{} 335 p := Payload{}
333 err := json.Unmarshal(load, &p) 336 err := json.Unmarshal(load, &p)
334 if err != nil { 337 if err != nil {
335 logger.Log("couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), rset.Row[0].(string)) 338 logger.Log("couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), rset.Row[0].(string))
336 } else { 339 } else {
337 metadata[e] = p 340 metadata[e] = p
338 } 341 }
339 } 342 }
340 stmt.Close() 343 stmt.Close()