Marko Tikvić / webutility

Download as
Browse Code »

Commit 44c82dbac6d62ac87b323ca5eef0d7ecfad4db89

Authored by Marko Tikvić
1 parent 3ad172fb64
Exists in master

simplified file reading

Showing 1 changed file with 3 additions and 11 deletions   Show diff stats
1 package webutility 1 package webutility
2 2
3 import ( 3 import (
4 "database/sql" 4 "database/sql"
5 "encoding/json" 5 "encoding/json"
6 "errors" 6 "errors"
7 "fmt" 7 "fmt"
8 "io" 8 "io"
9 "net/http" 9 "net/http"
10 "os" 10 "os"
11 "strings" 11 "strings"
12 "sync" 12 "sync"
13 "time" 13 "time"
14 14
15 "git.to-net.rs/marko.tikvic/gologger" 15 "git.to-net.rs/marko.tikvic/gologger"
16 ) 16 )
17 17
18 var ( 18 var (
19 mu = &sync.Mutex{} 19 mu = &sync.Mutex{}
20 metadata = make(map[string]Payload) 20 metadata = make(map[string]Payload)
21 21
22 updateQue = make(map[string][]byte) 22 updateQue = make(map[string][]byte)
23 23
24 metadataDB *sql.DB 24 metadataDB *sql.DB
25 activeProject string 25 activeProject string
26 26
27 inited bool 27 inited bool
28 driver string 28 driver string
29 logger *gologger.Logger 29 logger *gologger.Logger
30 ) 30 )
31 31
32 type LangMap map[string]map[string]string 32 type LangMap map[string]map[string]string
33 33
34 type Field struct { 34 type Field struct {
35 Parameter string `json:"param"` 35 Parameter string `json:"param"`
36 Type string `json:"type"` 36 Type string `json:"type"`
37 Visible bool `json:"visible"` 37 Visible bool `json:"visible"`
38 Editable bool `json:"editable"` 38 Editable bool `json:"editable"`
39 } 39 }
40 40
41 type CorrelationField struct { 41 type CorrelationField struct {
42 Result string `json:"result"` 42 Result string `json:"result"`
43 Elements []string `json:"elements"` 43 Elements []string `json:"elements"`
44 Type string `json:"type"` 44 Type string `json:"type"`
45 } 45 }
46 46
47 type Translation struct { 47 type Translation struct {
48 Language string `json:"language"` 48 Language string `json:"language"`
49 FieldsLabels map[string]string `json:"fieldsLabels"` 49 FieldsLabels map[string]string `json:"fieldsLabels"`
50 } 50 }
51 51
52 // output 52 // output
53 type PaginationLinks struct { 53 type PaginationLinks struct {
54 Base string `json:"base"` 54 Base string `json:"base"`
55 Next string `json:"next"` 55 Next string `json:"next"`
56 Prev string `json:"prev"` 56 Prev string `json:"prev"`
57 Self string `json:"self"` 57 Self string `json:"self"`
58 } 58 }
59 59
60 // input 60 // input
61 type PaginationParameters struct { 61 type PaginationParameters struct {
62 URL string `json:"-"` 62 URL string `json:"-"`
63 Offset int64 `json:"offset"` 63 Offset int64 `json:"offset"`
64 Limit int64 `json:"limit"` 64 Limit int64 `json:"limit"`
65 SortBy string `json:"sortBy"` 65 SortBy string `json:"sortBy"`
66 Order string `json:"order"` 66 Order string `json:"order"`
67 } 67 }
68 68
69 // TODO(marko) 69 // TODO(marko)
70 func GetPaginationParameters(req *http.Request) (p PaginationParameters) { 70 func GetPaginationParameters(req *http.Request) (p PaginationParameters) {
71 return p 71 return p
72 } 72 }
73 73
74 // TODO(marko) 74 // TODO(marko)
75 func (p *PaginationParameters) paginationLinks() (links PaginationLinks) { 75 func (p *PaginationParameters) paginationLinks() (links PaginationLinks) {
76 return links 76 return links
77 } 77 }
78 78
79 type Payload struct { 79 type Payload struct {
80 Method string `json:"method"` 80 Method string `json:"method"`
81 Params map[string]string `json:"params"` 81 Params map[string]string `json:"params"`
82 Lang []Translation `json:"lang"` 82 Lang []Translation `json:"lang"`
83 Fields []Field `json:"fields"` 83 Fields []Field `json:"fields"`
84 Correlations []CorrelationField `json:"correlationFields"` 84 Correlations []CorrelationField `json:"correlationFields"`
85 IdField string `json:"idField"` 85 IdField string `json:"idField"`
86 86
87 // Pagination 87 // Pagination
88 Count int64 `json:"count"` 88 Count int64 `json:"count"`
89 Total int64 `json:"total"` 89 Total int64 `json:"total"`
90 Links PaginationLinks `json:"_links"` 90 Links PaginationLinks `json:"_links"`
91 91
92 // Data holds JSON payload. It can't be used for itteration. 92 // Data holds JSON payload. It can't be used for itteration.
93 Data interface{} `json:"data"` 93 Data interface{} `json:"data"`
94 } 94 }
95 95
96 func (p *Payload) SetData(data interface{}) { 96 func (p *Payload) SetData(data interface{}) {
97 p.Data = data 97 p.Data = data
98 } 98 }
99 99
100 func (p *Payload) SetPaginationInfo(count, total int64, params PaginationParameters) { 100 func (p *Payload) SetPaginationInfo(count, total int64, params PaginationParameters) {
101 p.Count = count 101 p.Count = count
102 p.Total = total 102 p.Total = total
103 p.Links = params.paginationLinks() 103 p.Links = params.paginationLinks()
104 } 104 }
105 105
106 // NewPayload returs a payload sceleton for entity described with key. 106 // NewPayload returs a payload sceleton for entity described with key.
107 func NewPayload(r *http.Request, key string) Payload { 107 func NewPayload(r *http.Request, key string) Payload {
108 p := metadata[key] 108 p := metadata[key]
109 p.Method = r.Method + " " + r.RequestURI 109 p.Method = r.Method + " " + r.RequestURI
110 return p 110 return p
111 } 111 }
112 112
113 // DecodeJSON decodes JSON data from r to v. 113 // DecodeJSON decodes JSON data from r to v.
114 // Returns an error if it fails. 114 // Returns an error if it fails.
115 func DecodeJSON(r io.Reader, v interface{}) error { 115 func DecodeJSON(r io.Reader, v interface{}) error {
116 return json.NewDecoder(r).Decode(v) 116 return json.NewDecoder(r).Decode(v)
117 } 117 }
118 118
119 // InitPayloadsMetadata loads all payloads' information into 'metadata' variable. 119 // InitPayloadsMetadata loads all payloads' information into 'metadata' variable.
120 func InitPayloadsMetadata(drv string, db *sql.DB, project string) error { 120 func InitPayloadsMetadata(drv string, db *sql.DB, project string) error {
121 var err error 121 var err error
122 if drv != "ora" && drv != "mysql" { 122 if drv != "ora" && drv != "mysql" {
123 err = errors.New("driver not supported") 123 err = errors.New("driver not supported")
124 return err 124 return err
125 } 125 }
126 126
127 driver = drv 127 driver = drv
128 metadataDB = db 128 metadataDB = db
129 activeProject = project 129 activeProject = project
130 130
131 logger, err = gologger.New("metadata", gologger.MaxLogSize100KB) 131 logger, err = gologger.New("metadata", gologger.MaxLogSize100KB)
132 if err != nil { 132 if err != nil {
133 fmt.Printf("webutility: %s\n", err.Error()) 133 fmt.Printf("webutility: %s\n", err.Error())
134 } 134 }
135 135
136 mu.Lock() 136 mu.Lock()
137 defer mu.Unlock() 137 defer mu.Unlock()
138 err = initMetadata(project) 138 err = initMetadata(project)
139 if err != nil { 139 if err != nil {
140 return err 140 return err
141 } 141 }
142 inited = true 142 inited = true
143 143
144 return nil 144 return nil
145 } 145 }
146 146
147 func EnableHotloading(interval int) { 147 func EnableHotloading(interval int) {
148 if interval > 0 { 148 if interval > 0 {
149 go hotload(interval) 149 go hotload(interval)
150 } 150 }
151 } 151 }
152 152
153 func GetMetadataForAllEntities() map[string]Payload { 153 func GetMetadataForAllEntities() map[string]Payload {
154 return metadata 154 return metadata
155 } 155 }
156 156
157 func GetMetadataForEntity(t string) (Payload, bool) { 157 func GetMetadataForEntity(t string) (Payload, bool) {
158 p, ok := metadata[t] 158 p, ok := metadata[t]
159 return p, ok 159 return p, ok
160 } 160 }
161 161
162 func QueEntityModelUpdate(entityType string, v interface{}) { 162 func QueEntityModelUpdate(entityType string, v interface{}) {
163 updateQue[entityType], _ = json.Marshal(v) 163 updateQue[entityType], _ = json.Marshal(v)
164 } 164 }
165 165
166 func UpdateEntityModels(command string) (total, upd, add int, err error) { 166 func UpdateEntityModels(command string) (total, upd, add int, err error) {
167 if command != "force" && command != "missing" { 167 if command != "force" && command != "missing" {
168 return total, 0, 0, errors.New("webutility: unknown command: " + command) 168 return total, 0, 0, errors.New("webutility: unknown command: " + command)
169 } 169 }
170 170
171 if !inited { 171 if !inited {
172 return 0, 0, 0, errors.New("webutility: metadata not initialized but update was tried.") 172 return 0, 0, 0, errors.New("webutility: metadata not initialized but update was tried.")
173 } 173 }
174 174
175 total = len(updateQue) 175 total = len(updateQue)
176 176
177 toUpdate := make([]string, 0) 177 toUpdate := make([]string, 0)
178 toAdd := make([]string, 0) 178 toAdd := make([]string, 0)
179 179
180 for k, _ := range updateQue { 180 for k, _ := range updateQue {
181 if _, exists := metadata[k]; exists { 181 if _, exists := metadata[k]; exists {
182 if command == "force" { 182 if command == "force" {
183 toUpdate = append(toUpdate, k) 183 toUpdate = append(toUpdate, k)
184 } 184 }
185 } else { 185 } else {
186 toAdd = append(toAdd, k) 186 toAdd = append(toAdd, k)
187 } 187 }
188 } 188 }
189 189
190 var uStmt *sql.Stmt 190 var uStmt *sql.Stmt
191 if driver == "ora" { 191 if driver == "ora" {
192 uStmt, err = metadataDB.Prepare("update entities set entity_model = :1 where entity_type = :2") 192 uStmt, err = metadataDB.Prepare("update entities set entity_model = :1 where entity_type = :2")
193 if err != nil { 193 if err != nil {
194 logger.Trace(err.Error()) 194 logger.Trace(err.Error())
195 return 195 return
196 } 196 }
197 } else if driver == "mysql" { 197 } else if driver == "mysql" {
198 uStmt, err = metadataDB.Prepare("update entities set entity_model = ? where entity_type = ?") 198 uStmt, err = metadataDB.Prepare("update entities set entity_model = ? where entity_type = ?")
199 if err != nil { 199 if err != nil {
200 logger.Trace(err.Error()) 200 logger.Trace(err.Error())
201 return 201 return
202 } 202 }
203 } 203 }
204 for _, k := range toUpdate { 204 for _, k := range toUpdate {
205 _, err = uStmt.Exec(string(updateQue[k]), k) 205 _, err = uStmt.Exec(string(updateQue[k]), k)
206 if err != nil { 206 if err != nil {
207 logger.Trace(err.Error()) 207 logger.Trace(err.Error())
208 return 208 return
209 } 209 }
210 upd++ 210 upd++
211 } 211 }
212 212
213 blankPayload, _ := json.Marshal(Payload{}) 213 blankPayload, _ := json.Marshal(Payload{})
214 var iStmt *sql.Stmt 214 var iStmt *sql.Stmt
215 if driver == "ora" { 215 if driver == "ora" {
216 iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(:1, :2, :3, :4)") 216 iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(:1, :2, :3, :4)")
217 if err != nil { 217 if err != nil {
218 logger.Trace(err.Error()) 218 logger.Trace(err.Error())
219 return 219 return
220 } 220 }
221 } else if driver == "mysql" { 221 } else if driver == "mysql" {
222 iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(?, ?, ?, ?)") 222 iStmt, err = metadataDB.Prepare("insert into entities(projekat, metadata, entity_type, entity_model) values(?, ?, ?, ?)")
223 if err != nil { 223 if err != nil {
224 logger.Trace(err.Error()) 224 logger.Trace(err.Error())
225 return 225 return
226 } 226 }
227 } 227 }
228 for _, k := range toAdd { 228 for _, k := range toAdd {
229 _, err = iStmt.Exec(activeProject, string(blankPayload), k, string(updateQue[k])) 229 _, err = iStmt.Exec(activeProject, string(blankPayload), k, string(updateQue[k]))
230 if err != nil { 230 if err != nil {
231 logger.Trace(err.Error()) 231 logger.Trace(err.Error())
232 return 232 return
233 } 233 }
234 metadata[k] = Payload{} 234 metadata[k] = Payload{}
235 add++ 235 add++
236 } 236 }
237 237
238 return total, upd, add, nil 238 return total, upd, add, nil
239 } 239 }
240 240
241 func initMetadata(project string) error { 241 func initMetadata(project string) error {
242 rows, err := metadataDB.Query(`select 242 rows, err := metadataDB.Query(`select
243 entity_type, 243 entity_type,
244 metadata 244 metadata
245 from entities 245 from entities
246 where projekat = ` + fmt.Sprintf("'%s'", project)) 246 where projekat = ` + fmt.Sprintf("'%s'", project))
247 if err != nil { 247 if err != nil {
248 return err 248 return err
249 } 249 }
250 defer rows.Close() 250 defer rows.Close()
251 251
252 if len(metadata) > 0 { 252 if len(metadata) > 0 {
253 metadata = nil 253 metadata = nil
254 } 254 }
255 metadata = make(map[string]Payload) 255 metadata = make(map[string]Payload)
256 for rows.Next() { 256 for rows.Next() {
257 var name, load string 257 var name, load string
258 rows.Scan(&name, &load) 258 rows.Scan(&name, &load)
259 259
260 p := Payload{} 260 p := Payload{}
261 err := json.Unmarshal([]byte(load), &p) 261 err := json.Unmarshal([]byte(load), &p)
262 if err != nil { 262 if err != nil {
263 logger.Log("webutility: couldn't init: '%s' metadata: %s:\n%s\n", name, err.Error(), load) 263 logger.Log("webutility: couldn't init: '%s' metadata: %s:\n%s\n", name, err.Error(), load)
264 } else { 264 } else {
265 metadata[name] = p 265 metadata[name] = p
266 } 266 }
267 } 267 }
268 268
269 return nil 269 return nil
270 } 270 }
271 271
272 func LoadMetadataFromFile(path string) error { 272 func LoadMetadataFromFile(path string) error {
273 lines, err := ReadFileLines(path) 273 lines, err := ReadFileLines(path)
274 if err != nil { 274 if err != nil {
275 return err 275 return err
276 } 276 }
277 277
278 if len(metadata) > 0 { 278 if len(metadata) > 0 {
279 metadata = nil 279 metadata = nil
280 } 280 }
281 metadata = make(map[string]Payload) 281 metadata = make(map[string]Payload)
282 282
283 var name string 283 var name string
284 for i, l := range lines { 284 for i, l := range lines {
285 fmt.Println(l)
285 l = strings.TrimSpace(l) 286 l = strings.TrimSpace(l)
286 if len(l) == 0 { 287 if len(l) == 0 {
287 continue 288 continue
288 } 289 }
289 290
290 if strings.HasPrefix(l, "[") { 291 if strings.HasPrefix(l, "[") {
291 name = strings.Trim(l, "[]") 292 name = strings.Trim(l, "[]")
292 metadata[name] = Payload{ 293 metadata[name] = Payload{
293 Lang: []Translation{ 294 Lang: []Translation{
294 Translation{ 295 Translation{
295 Language: "sr", 296 Language: "sr",
296 FieldsLabels: make(map[string]string), 297 FieldsLabels: make(map[string]string),
297 }, 298 },
298 }, 299 },
299 } 300 }
300 continue 301 continue
301 } 302 }
302 303
303 if name == "" { 304 if name == "" {
304 return fmt.Errorf("webutility: LoadMetadataFromFile error on line %d: [no header][%s]\n", i+1, l) 305 return fmt.Errorf("webutility: LoadMetadataFromFile error on line %d: [no header][%s]\n", i+1, l)
305 } 306 }
306 307
307 parts := strings.Split(l, ":") 308 parts := strings.Split(l, ":")
308 if len(parts) < 2 { 309 if len(parts) < 2 {
309 return fmt.Errorf("webutility: LoadMetadataFromFile error on line %d: [invalid format][%s]\n", i+1, l) 310 return fmt.Errorf("webutility: LoadMetadataFromFile error on line %d: [invalid format][%s]\n", i+1, l)
310 } 311 }
311 312
312 k := strings.TrimSpace(parts[0]) 313 k := strings.TrimSpace(parts[0])
313 v := strings.TrimSpace(parts[1]) 314 v := strings.TrimSpace(parts[1])
314 if v != "-" { 315 if v != "-" {
315 metadata[name].Lang[0].FieldsLabels[k] = v 316 metadata[name].Lang[0].FieldsLabels[k] = v
316 } 317 }
317 } 318 }
318 319
319 return nil 320 return nil
320 } 321 }
321 322
322 func ReadFileLines(path string) ([]string, error) { 323 func ReadFileLines(path string) ([]string, error) {
323 f, err := os.Open(path) 324 f, err := os.Open(path)
324 if err != nil { 325 if err != nil {
325 return nil, err 326 return nil, err
326 } 327 }
327 defer f.Close() 328 defer f.Close()
328 329
329 var s strings.Builder 330 var s strings.Builder
330 for {
331 buf := make([]byte, 1024)
332 n, err := f.Read(buf)
333 if err != nil {
334 if err == io.EOF {
335 break
336 }
337 331
338 return nil, err 332 if _, err = io.Copy(&s, f); err != nil {
339 } 333 return nil, err
340
341 s.Write(buf[:n])
342 } 334 }
343 335
344 lines := strings.Split(s.String(), "\n") 336 lines := strings.Split(s.String(), "\n")
345 337
346 return lines, nil 338 return lines, nil
347 } 339 }
348 340
349 func hotload(n int) { 341 func hotload(n int) {
350 entityScan := make(map[string]int64) 342 entityScan := make(map[string]int64)
351 firstCheck := true 343 firstCheck := true
352 for { 344 for {
353 time.Sleep(time.Duration(n) * time.Second) 345 time.Sleep(time.Duration(n) * time.Second)
354 rows, err := metadataDB.Query(`select 346 rows, err := metadataDB.Query(`select
355 ora_rowscn, 347 ora_rowscn,
356 entity_type 348 entity_type
357 from entities where projekat = ` + fmt.Sprintf("'%s'", activeProject)) 349 from entities where projekat = ` + fmt.Sprintf("'%s'", activeProject))
358 if err != nil { 350 if err != nil {
359 logger.Log("webutility: hotload failed: %v\n", err) 351 logger.Log("webutility: hotload failed: %v\n", err)
360 time.Sleep(time.Duration(n) * time.Second) 352 time.Sleep(time.Duration(n) * time.Second)
361 continue 353 continue
362 } 354 }
363 355
364 var toRefresh []string 356 var toRefresh []string
365 for rows.Next() { 357 for rows.Next() {
366 var scanID int64 358 var scanID int64
367 var entity string 359 var entity string
368 rows.Scan(&scanID, &entity) 360 rows.Scan(&scanID, &entity)
369 oldID, ok := entityScan[entity] 361 oldID, ok := entityScan[entity]
370 if !ok || oldID != scanID { 362 if !ok || oldID != scanID {
371 entityScan[entity] = scanID 363 entityScan[entity] = scanID
372 toRefresh = append(toRefresh, entity) 364 toRefresh = append(toRefresh, entity)
373 } 365 }
374 } 366 }
375 rows.Close() 367 rows.Close()
376 368
377 if rows.Err() != nil { 369 if rows.Err() != nil {
378 logger.Log("webutility: hotload rset error: %v\n", rows.Err()) 370 logger.Log("webutility: hotload rset error: %v\n", rows.Err())
379 time.Sleep(time.Duration(n) * time.Second) 371 time.Sleep(time.Duration(n) * time.Second)
380 continue 372 continue
381 } 373 }
382 374
383 if len(toRefresh) > 0 && !firstCheck { 375 if len(toRefresh) > 0 && !firstCheck {
384 mu.Lock() 376 mu.Lock()
385 refreshMetadata(toRefresh) 377 refreshMetadata(toRefresh)
386 mu.Unlock() 378 mu.Unlock()
387 } 379 }
388 if firstCheck { 380 if firstCheck {
389 firstCheck = false 381 firstCheck = false
390 } 382 }
391 } 383 }
392 } 384 }
393 385
394 func refreshMetadata(entities []string) { 386 func refreshMetadata(entities []string) {
395 for _, e := range entities { 387 for _, e := range entities {
396 fmt.Printf("refreshing %s\n", e) 388 fmt.Printf("refreshing %s\n", e)
397 rows, err := metadataDB.Query(`select 389 rows, err := metadataDB.Query(`select
398 metadata 390 metadata
399 from entities 391 from entities
400 where projekat = ` + fmt.Sprintf("'%s'", activeProject) + 392 where projekat = ` + fmt.Sprintf("'%s'", activeProject) +
401 ` and entity_type = ` + fmt.Sprintf("'%s'", e)) 393 ` and entity_type = ` + fmt.Sprintf("'%s'", e))
402 394
403 if err != nil { 395 if err != nil {
404 logger.Log("webutility: refresh: prep: %v\n", err) 396 logger.Log("webutility: refresh: prep: %v\n", err)
405 rows.Close() 397 rows.Close()
406 continue 398 continue
407 } 399 }
408 400
409 for rows.Next() { 401 for rows.Next() {
410 var load string 402 var load string
411 rows.Scan(&load) 403 rows.Scan(&load)
412 p := Payload{} 404 p := Payload{}
413 err := json.Unmarshal([]byte(load), &p) 405 err := json.Unmarshal([]byte(load), &p)
414 if err != nil { 406 if err != nil {
415 logger.Log("webutility: couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), load) 407 logger.Log("webutility: couldn't refresh: '%s' metadata: %s\n%s\n", e, err.Error(), load)
416 } else { 408 } else {
417 metadata[e] = p 409 metadata[e] = p
418 } 410 }
419 } 411 }
420 rows.Close() 412 rows.Close()
421 } 413 }
422 } 414 }
423 415
424 /* 416 /*
425 func ModifyMetadataForEntity(entityType string, p *Payload) error { 417 func ModifyMetadataForEntity(entityType string, p *Payload) error {
426 md, err := json.Marshal(*p) 418 md, err := json.Marshal(*p)
427 if err != nil { 419 if err != nil {
428 return err 420 return err
429 } 421 }
430 422
431 mu.Lock() 423 mu.Lock()
432 defer mu.Unlock() 424 defer mu.Unlock()
433 _, err = metadataDB.PrepAndExe(`update entities set 425 _, err = metadataDB.PrepAndExe(`update entities set
434 metadata = :1 426 metadata = :1
435 where projekat = :2 427 where projekat = :2
436 and entity_type = :3`, 428 and entity_type = :3`,
437 string(md), 429 string(md),
438 activeProject, 430 activeProject,
439 entityType) 431 entityType)
440 if err != nil { 432 if err != nil {
441 return err 433 return err
442 } 434 }
443 return nil 435 return nil
444 } 436 }
445 437
446 func DeleteEntityModel(entityType string) error { 438 func DeleteEntityModel(entityType string) error {
447 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType) 439 _, err := metadataDB.PrepAndExe("delete from entities where entity_type = :1", entityType)
448 if err == nil { 440 if err == nil {
449 mu.Lock() 441 mu.Lock()
450 delete(metadata, entityType) 442 delete(metadata, entityType)
451 mu.Unlock() 443 mu.Unlock()
452 } 444 }
453 return err 445 return err
454 } 446 }
455 */ 447 */